def metadata_delete(gvar): """ Delete a group metadata file. """ mandatory = ['-mn'] required = [] optional = ['-g', '-H', '-h', '-s', '-xA', '-Y'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Check that the target groupmetadata file exists. response = requests(gvar, '/group/list/') _found = False for row in response['group_list']: if row['group_name'] == gvar['active_group']: metadata_names = row['metadata_names'].split(',') for metadata_name in metadata_names: if row['group_name'] == gvar['active_group']: _found = True break if not _found: print( 'Error: "%s group metadata-delete" cannot delete "%s::%s", file doesn\'t exist.' % (gvar['command_name'], response['active_group'], gvar['user_settings']['metadata-name'])) exit(1) # Confirm group metadata file delete. if not gvar['user_settings']['yes']: print( 'Are you sure you want to delete the metadata file "%s::%s"? (yes|..)' % (response['active_group'], gvar['user_settings']['metadata-name'])) _reply = input() if _reply != 'yes': print('%s group metadata-delete "%s::%s" cancelled.' % (gvar['command_name'], response['active_group'], gvar['user_settings']['metadata-name'])) exit(0) # Delete the group metadata file. response = requests(gvar, '/group/metadata-delete/', form_data={ 'metadata_name': gvar['user_settings']['metadata-name'], }) if response['message']: print(response['message'])
def base_metadata_save(gvar, context=None): """ Save the specified cloud or cloud metadata file. """ if context is None: print('Error: This code should not have been called without a context') traceback.print_tb() return if context == 'cloud': mandatory = ['-cn', '-mn', '-f'] else: mandatory = ['-mn', '-f'] required = [] optional = ['-F', '-g', '-H', '-h', '-s', '-v', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). if context == 'cloud': response = requests(gvar, '/cloud/metadata-fetch/', query_data={ 'cloud_name': gvar['user_settings']['cloud-name'], 'metadata_name': gvar['user_settings']['metadata-name'] }) else: response = requests(gvar, '/group/metadata-fetch/', query_data={ 'metadata_name': gvar['user_settings']['metadata-name'] }) if os.path.exists(gvar['user_settings'] ['file-path']) and not gvar['user_settings']['force']: print( 'Error: The specified metadata file "%s" already exists, use -F to overwrite .' % gvar['user_settings']['file-path']) exit(1) # Write the reference copy. fd = open(gvar['user_settings']['file-path'], 'w') fd.write(response['metadata']) fd.close()
def add(gvar): """ Add a user. """ mandatory = ['-un', '-upw'] required = [] optional = ['-g', '-gn', '-H', '-h', '-SU', '-s', '-ucn', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys( gvar, mandatory, required, optional, key_map=KEY_MAP) # Create the user. response = requests( gvar, '/user/add/', form_data ) if response['message']: print(response['message'])
def update(gvar): """ Modify the specified group. """ mandatory = ['-gn'] required = [] optional = ['-g', '-gm', '-H', '-h', '-s', '-un', '-uo', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) if len(form_data) < 2: print( 'Error: "%s group update" requires at least one option to update.' % gvar['command_name']) exit(1) # Create the group. response = requests(gvar, '/group/update/', form_data) if response['message']: print(response['message'])
def metadata_update(gvar): """ Update metadata fiel information. """ mandatory = ['-mn'] required = [] optional = ['-g', '-H', '-h', '-me', '-mmt', '-mp', '-s', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) if len(form_data) < 2: print( 'Error: "%s group metadata-update" requires at least one option to modify.' % gvar['command_name']) exit(1) # Create the cloud. response = requests(gvar, '/group/metadata-update/', form_data) if response['message']: print(response['message'])
def update(gvar): """ Modify a cloud in the active group. """ mandatory = ['-cn'] required = [] optional = ['-ca', '-ce', '-cfe', '-cfo', '-cpw', '-cP', '-cp', '-cr', '-csp', '-ct', '-cU', '-cu', '-g', '-ga', '-gme', '-gmo', '-H', '-h', '-s', '-vc', '-vcs', '-vf', '-vi', '-vk', '-vka', '-vk', '-vn', '-vr', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys( gvar, mandatory, required, optional, key_map=KEY_MAP) if len(form_data) < 2: print('Error: "%s cloud update" requires at least one option to modify.' % gvar['command_name']) exit(1) # Create the cloud. response = requests( gvar, '/cloud/update/', form_data ) if response['message']: print(response['message'])
def add(gvar): """ Add a cloud to the active group. """ mandatory = ['-ca', '-cn', '-cp', '-cpw', '-cr', '-ct', '-cu'] required = [] optional = ['-ce', '-cfe', '-cP', '-csp', '-cU', '-g', '-ga', '-gme', '-H', '-h', '-s', '-vc', '-vcs', '-vf', '-vi', '-vk', '-vka', '-vk', '-vn', '-vr', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys( gvar, mandatory, required, optional, key_map=KEY_MAP) # Create the cloud. response = requests( gvar, '/cloud/add/', form_data ) if response['message']: print(response['message'])
def delete(gvar): """ Delete a cloud from the active group. """ mandatory = ['-cn'] required = [] optional = ['-g', '-H', '-h', '-s', '-xA', '-Y'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Check that the target cloud exists. response = requests(gvar, '/cloud/list/') _found = False for row in response['cloud_list']: if row['cloud_name'] == gvar['user_settings']['cloud-name']: _found = True break if not _found: print('Error: "%s cloud delete" cannot delete "%s", cloud doesn\'t exist in group "%s".' % (gvar['command_name'], gvar['user_settings']['cloud-name'], response['active_group'])) exit(1) # Confirm cloud delete. if not gvar['user_settings']['yes']: print('Are you sure you want to delete cloud "%s::%s"? (yes|..)' % (response['active_group'], gvar['user_settings']['cloud-name'])) _reply = input() if _reply != 'yes': print('%s cloud delete "%s::%s" cancelled.' % (gvar['command_name'], response['active_group'], gvar['user_settings']['cloud-name'])) exit(0) # Delete the cloud. response = requests( gvar, '/cloud/delete/', form_data = { 'cloud_name': gvar['user_settings']['cloud-name'] } ) if response['message']: print(response['message'])
def delete(gvar): """ Delete a user. """ mandatory = ['-un'] required = [] optional = ['-g', '-H', '-h', '-s', '-xA', '-Y'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Check that the target user exists. response = requests(gvar, '/user/list/') _found = False for row in response['user_list']: if row['username'] == gvar['user_settings']['username']: _found = True break if not _found: print('Error: "%s user delete" cannot delete "%s", user doesn\'t exist.' % (gvar['command_name'], gvar['user_settings']['username'])) exit(1) # Confirm user delete. if not gvar['user_settings']['yes']: print('Are you sure you want to delete user "%s"? (yes|..)' % gvar['user_settings']['username']) _reply = input() if _reply != 'yes': print('%s user delete "%s" cancelled.' % (gvar['command_name'], gvar['user_settings']['username'])) exit(0) # Delete the user. response = requests( gvar, '/user/delete/', form_data = { 'username': gvar['user_settings']['username'] } ) if response['message']: print(response['message'])
def defaults(gvar): """ Modify the specified group defaults. """ mandatory = [] required = [] optional = [ '-CSEP', '-CSV', '-g', '-H', '-h', '-htcf', '-htcn', '-htcu', '-jc', '-jd', '-jr', '-js', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-vf', '-vi', '-vka', '-vk', '-vn', '-xA' ] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) # List the current defaults. If the form_data contains any optional fields, # those values will be updated before the list is retrieved. response = requests(gvar, '/group/defaults/', form_data) if response['message']: print(response['message']) # Print report show_active_user_groups(gvar, response) show_table( gvar, response['defaults_list'], [ 'group_name/Group,k', 'htcondor_fqdn/FQDN/HTCondor', 'htcondor_name/Name/HTCondor', 'htcondor_other_submitters/Users/HTCondor', 'vm_flavor/Flavor/VM', 'vm_image/Image/VM', 'vm_keep_alive/Keep Alive/VM', 'vm_keyname/Keyname/VM', 'vm_network/Network/VM', 'job_cpus/Cores/Job', 'job_disk/Disk (GBs)/Job', 'job_ram/RAM (MBs)/Job', 'job_swap/Swap (GBs)/Job', ], title="Active Group Defaults:", )
def config(gvar): """ Modify the specified group defaults. """ mandatory = [] required = [] optional = [ '-c', '-cc', '-CSEP', '-CSV', '-dci', '-djg', '-eg', '-g', '-H', '-h', '-lf', '-ll', '-nld', '-NV', '-ok', '-r', '-s', '-sic', '-siC', '-sif', '-sii', '-sij', '-sik', '-sil', '-sim', '-sin', '-siv', '-sml', '-sms', '-V', '-VC', '-xA' ] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) # List the current defaults. If the form_data contains any optional fields, # those values will be updated before the list is retrieved. response = requests(gvar, '/server/config/', form_data) if response['message']: print(response['message']) # Print report show_active_user_groups(gvar, response) show_table( gvar, response['config_list'], [ 'category/Category,k', 'config_key/Config Key,k', 'config_type/Type', 'config_value/Value', ], title="Server Configuration:", )
def metadata_list(gvar): """ List clouds for the active group. """ mandatory = [] required = [] optional = [ '-CSEP', '-CSV', '-g', '-H', '-h', '-mn', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-xA' ] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/group/metadata-list/') if response['message']: print(response['message']) # Filter response as requested (or not). group_metadata_list = _filter_by_group_name_and_or_metadata_name( gvar, response['group_metadata_list']) # Print report. show_active_user_groups(gvar, response) show_table( gvar, group_metadata_list, [ 'group_name/Group,k', 'metadata_name/Metadata Filename,k', 'enabled/Enabled', 'priority/Priority', 'mime_type/MIME Type', ], title="Active Group/Metadata:", )
def list(gvar): """ List users. """ mandatory = [] required = [] optional = ['-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-ok', '-r', '-s', '-un', '-V', '-VC', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the user). response = requests(gvar, '/user/list/') if response['message']: print(response['message']) # Filter response as requested (or not). user_list = _filter_by_user(gvar, response['user_list']) # Print report show_active_user_groups(gvar, response) show_table( gvar, user_list, [ 'username/Username,k', 'cert_cn/Common Name', 'active_group/Active Group', 'user_groups/User Groups', 'available_groups/Not In Groups', 'is_superuser/Super User', 'join_date/Joined', ], title="Users:", )
def list(gvar): """ List groups. """ mandatory = [] required = [] optional = [ '-CSEP', '-CSV', '-g', '-gn', '-H', '-h', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-xA' ] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/group/list/') if response['message']: print(response['message']) # Filter response as requested (or not). group_list = _filter_by_group_name_and_or_metadata_name( gvar, response['group_list']) # Print report show_active_user_groups(gvar, response) show_table( gvar, group_list, [ 'group_name/Group,k', 'condor_central_manager/Central Manager', 'metadata_names/Metadata Filenames', ], title="Groups:", )
def metadata_load(gvar): """ Load a new group metadata file. """ mandatory = ['-f', '-mn'] required = [] optional = ['-g', '-H', '-h', '-me', '-mmt', '-mp', '-s', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) if not os.path.exists(gvar['user_settings']['file-path']): print('Error: The specified metadata file "%s" does not exist.' % gvar['user_settings']['file-path']) exit(1) # # Verify the changed metadata file and build input form data. # form_data = { # **verify_yaml_file(gvar['user_settings']['file-path']), # 'metadata_name': gvar['user_settings']['metadata-name'], # } # Replace the metadata file. response = requests(gvar, '/group/metadata-add/', { **form_data, **verify_yaml_file(gvar['user_settings']['file-path']), }) if response['message']: print(response['message'])
def settings(gvar): """ Modify the specified user. """ mandatory = [] required = [] optional = ['-H', '-h', '-s', '-upw', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) if len(form_data) < 1: print( 'Error: "%s my settings" requires at least one option to update.' % gvar['command_name']) exit(1) # Create the user. response = requests(gvar, '/user/settings/', form_data) if response['message']: print(response['message']) if 'server' not in gvar['command_args']: gvar['command_args']['server'] = gvar['server'] gvar['user_settings']['server-password'] = gvar['user_settings'][ 'user-password'] del gvar['user_settings']['user-password'] csv2_defaults.set(gvar)
def update(gvar): """ Modify a VM in the active group. """ mandatory = ['-vh', '-vo'] required = [] optional = ['-cn', '-g', '-H', '-h', '-s', '-vS', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. form_data = check_keys(gvar, mandatory, required, optional, key_map=KEY_MAP) # Create the cloud. response = requests(gvar, '/vm/update/', form_data) if response['message']: print(response['message'])
def metadata_edit(gvar): """ Edit the specified group metadata file. """ mandatory = ['-mn'] required = ['-te'] optional = ['-g', '-H', '-h', '-s', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests( gvar, '/group/metadata-fetch/%s/' % gvar['user_settings']['metadata-name']) # Ensure the fetch directory structure exists. fetch_dir = '%s/.csv2/%s/files/%s/metadata' % ( gvar['home_dir'], gvar['server'], response['group_name'], ) if not os.path.exists(fetch_dir): os.makedirs(fetch_dir, mode=0o700) # Write the reference copy. fd = open('%s/.%s' % (fetch_dir, response['metadata_name']), 'w') # fd.write('# metadata_enabled: %s, metadata_mime_type: %s\n%s' % (response['metadata_enabled'], response['metadata_mime_type'], response['metadata'])) fd.write(response['metadata']) fd.close() # Write the edit copy. fd = open('%s/%s' % (fetch_dir, response['metadata_name']), 'w') # fd.write('# metadata_enabled: %s, metadata_mime_type: %s\n%s' % (response['metadata_enabled'], response['metadata_mime_type'], response['metadata'])) fd.write(response['metadata']) fd.close() # Edit the metadata file. editor = '' if 'text-editor' in gvar['user_settings']: editor = gvar['user_settings']['text-editor'] else: editor = os.getenv('EDITOR') p = Popen([editor, '%s/%s' % (fetch_dir, response['metadata_name'])]) p.communicate() if filecmp.cmp('%s/.%s' % (fetch_dir, response['metadata_name']), '%s/%s' % (fetch_dir, response['metadata_name'])): print('%s group metadata-edit "%s::%s" completed, no changes.' % (gvar['command_name'], response['group_name'], response['metadata_name'])) exit(0) # Verify the changed metadata file. form_data = { **verify_yaml_file('%s/%s' % (fetch_dir, response['metadata_name'])), 'metadata_name': response['metadata_name'], } # Replace the metadata file. response = requests(gvar, '/group/metadata-update/', form_data) if response['message']: print(response['message'])
def list(gvar): """ List VMs for the active group. """ mandatory = [] required = [] optional = [ '-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-vc', '-vd', '-vF', '-vf', '-vh', '-vr', '-vS', '-vs', '-xA' ] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/vm/list/%s' % _selector(gvar)) if response['message']: print(response['message']) # Filter response as requested (or not). vm_list = _filter(gvar, response['vm_list']) # Print report. show_active_user_groups(gvar, response) show_table( gvar, vm_list, [ 'group_name/Group,k', 'cloud_name/Cloud,k', 'hostname/Hostname,k', 'vmid/VMID', 'vm_ips/IPs', 'vm_floating_ips/Floating IPs', 'auth_url/Authorization URL', 'project/Project', 'status/Status', 'flavor_id/Flavor ID', 'task/Task', 'power_status/Power Status', 'start_time/Start Time', 'retire_request_time/Retire Request Time', 'retired_time/Retired Time', 'terminate/Terminate', 'terminate_time/Terminate Time', 'status_changed_time/Status Change Time', 'last_updated/Last Updated', 'flavor_name/Flavor', 'condor_slots/Condor Slots', 'foreign_vm/Foreign', 'cores/cores', 'disk/Disk (GBs)', 'ram/Ram (MBs)', 'swap/Swap (GBs)', 'poller_status/Poller Status', 'manual_control/Manual_Control', ], title="VMs:", )
def _conditionally_restore_a_file(gvar, path, metadata): """ Compare the metadata on the server with its' backup and if they differ, ask the user whether they want to restore the backup. """ if metadata and path in gvar['backup_path_list']: del gvar['backup_path_list'][path] path_dir = os.path.dirname(path) if path[:-1] == path_dir: print('Skipping metadata file with no name, path="%s"' % path) return work_path = '%s/work%s' % (gvar['temp_dir'], path) work_dir = os.path.dirname(work_path) if not os.path.exists(work_dir): os.makedirs(work_dir) fd = open(work_path, 'w') if metadata: fd.write(json.dumps(metadata)) fd.close() if 'backup-key' in gvar['user_settings']: compare_path = '%s/compare%s' % (gvar['temp_dir'], path) compare_dir = os.path.dirname(compare_path) if not os.path.exists(compare_dir): os.makedirs(compare_dir) restore_path = compare_path p = Popen( [ 'ansible-vault', 'decrypt', path, '--output', restore_path, '--vault-password-file', gvar['user_settings']['backup-key'] ], stdout=PIPE, stderr=PIPE ) stdout, stderr = p.communicate() if p.returncode != 0: raise Exception('Error: ansible-vault decrypt: stdout=%s, stderr=%s' % (stdout, stderr)) else: restore_path = path if not metadata or not filecmp.cmp(work_path, restore_path, shallow=False): while(True): if metadata: rsp = input('Backup up differs from server metadata, path=%s. Do you want to restore the backup? (yes, no (default), show, or quit):' % path) else: rsp = input('File missing from server, backup file path=%s. Do you want to restore the backup? (yes, no (default), show, or quit):' % path) if rsp == '' or 'no'[:len(rsp)] == rsp.lower(): break elif 'yes'[:len(rsp)] == rsp.lower(): words = restore_path.split('/') ix = words.index('groups') response = requests(gvar, '/settings/prepare/', {'group': words[ix+1]}) fd = open(restore_path) restore_data = qc(json.loads(fd.read()), columns=['group_name'], option='prune') fd.close() if len(words) == ix+3: response = requests(gvar, '/group/defaults/', _get_restore_data(gvar, restore_path, other_prune_columns=['job_scratch'])) elif len(words) == ix+4: response = requests(gvar, '/group/metadata-update/', _get_restore_data(gvar, restore_path)) elif len(words) == ix+5: response = requests(gvar, '/cloud/update/', _get_restore_data(gvar, restore_path)) elif len(words) == ix+6: response = requests(gvar, '/cloud/metadata-update/', _get_restore_data(gvar, restore_path)) if response['response_code'] == 0: print('Successfully restored.') else: raise Exception('Metadata restoration failed, msg=%s' % response['message']) break elif 'quit'[:len(rsp)] == rsp.lower(): exit(0) elif 'show'[:len(rsp)] == rsp.lower(): expanded_work_path = _expand(gvar, work_path) expanded_restore_path = _expand(gvar, restore_path) print('< server (%s), > backup' % gvar['server']) p = Popen( [ 'diff', '-y', expanded_work_path, expanded_restore_path ] ) p.communicate() else: print('Invalid response "%s".' % rsp)
def list(gvar): """ List clouds for the active group. """ mandatory = [] required = [] optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-jh', '-jI', '-ji', '-jp', '-jR', '-jrc', '-jrd', '-jrr', '-jrs', '-jS', '-jtc', '-ju', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys( gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/job/list/') if response['message']: print(response['message']) # Filter response as requested (or not). job_list = _filter(gvar, response['job_list']) # Print report. show_active_user_groups(gvar, response) show_table( gvar, job_list, [ 'group_name/Group,k', 'global_job_id/Job ID,k', 'cluster_id/Cluster ID', 'proc_id/Process ID', 'user/User', 'user_data/User Data', 'requirements/Requirements', 'target_clouds/Target Clouds', 'cloud_name/Cloud', 'instance_type/Instance Type', 'request_cpus/CPUs/Requested', 'request_ram/RAM {MBs}/Requested', 'request_disk/Disk {GBs}/Requested', 'request_swap/Swap (GBs)/Requested', 'job_per_core/Jobs per Core', 'image/Image', 'network/Network', 'job_priority/Priority/Job', 'job_status/Status Code/Job', 'js_idle/Idle/Job Status Flags', 'js_running/Running/Job Status Flags', 'js_completed/Completed/Job Status Flags', 'js_held/Held/Job Status Flags', 'js_other/Other/Job Status Flags', 'keep_alive/Keep Alive (seconds)', 'max_price/Max Spot Price', 'entered_current_status/State Change Date', 'q_date/Queued Date', 'held_reason/Held Job Reason', ], title="Jobs", )
def backup(gvar): """ Backup all user data for all groups/clouds for each server configured in the user defaults. """ mandatory = [] required = ['-br'] optional = ['-bk', '-xA'] servers = {} if gvar['retrieve_options']: return mandatory + required + optional # Retrieve the backup repository and all server information. servers['settings'], servers['xref'] = _get_repository_and_servers(gvar) # Check for missing arguments or help required. check_keys( gvar, mandatory, required, optional) # If the backup directory is an encrypted git repository, create a working temorary directory. _update_git(gvar, 'pre-backup') # If the backup directory is an encrypted git repository, create a working temorary directory. if 'backup-key' in gvar['user_settings'] and os.path.exists('%s/.git' % gvar['user_settings']['backup-repository']): gvar['temp_dir'] = mkdtemp() else: gvar['temp_dir'] = None # Retrieve data to backup for each cloudscheduler server. fetched = {} for server in sorted(servers['settings']): host, host_dir = _set_host(gvar, servers, server) if not host: print('Skipping server="%s", no server address.' % server) continue if host not in fetched: fetched[host] = {} # Save the initital server group so it can be restored later. response = requests(gvar, '/settings/prepare/') servers['initial_server_group'] = gvar['active_group'] groups = gvar['user_groups'] for group in sorted(groups): if group in fetched[host]: continue fetched[host][group] = True # Switch groups. response = requests(gvar, '/settings/prepare/', {'group': group}) print('Fetching: server=%s, group=%s' % (server, group)) response = requests(gvar, '/group/defaults/') _create_backup_file(gvar, '%s/groups/%s/defaults' % (host_dir, response['defaults_list'][0]['group_name']), response['defaults_list'][0]) response = requests(gvar, '/group/metadata-list/') for metadata in response['group_metadata_list']: _create_backup_file(gvar, '%s/groups/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['metadata_name']), metadata) response = requests(gvar, '/cloud/list/') for cloud in response['cloud_list']: _create_backup_file(gvar, '%s/groups/%s/clouds/%s/settings' % (host_dir, cloud['group_name'], cloud['cloud_name']), _get_cloud_settings(cloud)) response = requests(gvar, '/cloud/metadata-list/') for metadata in response['cloud_metadata_list']: _create_backup_file(gvar, '%s/groups/%s/clouds/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['cloud_name'], metadata['metadata_name']), metadata) # Restore the server's initial group. response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']}) _update_git(gvar, 'post-backup') if gvar['temp_dir']: rmtree(gvar['temp_dir'])
def restore(gvar): """ Restore user data. """ mandatory = [] required = ['-br'] optional = ['-bk', '-xA'] servers = {} if gvar['retrieve_options']: return mandatory + required + optional # Retrieve the backup repository and all server information. servers['settings'], servers['xref'] = _get_repository_and_servers(gvar) # Check for missing arguments or help required. check_keys( gvar, mandatory, required, optional) # Commit any outstanding repository updates prior to the restore. _update_git(gvar, 'pre-restore') # Retrieve an inventory of the current backup files. gvar['backup_path_list'] = {} _get_backup_path_list(gvar, gvar['user_settings']['backup-repository']) # We need a temporary working directory. gvar['temp_dir'] = mkdtemp() # Retrieve data from each cloudscheduler server to compare with its' backup. If the # backup is different, conditionally (allow the user to choose) restore it. fetched = {} host_group_xref = {} for server in sorted(servers['settings']): host, host_dir = _set_host(gvar, servers, server) if not host: print('Skipping server="%s", no server address.' % server) continue if host not in fetched: fetched[host] = {} # Save the initital server group so it can be restored later. response = requests(gvar, '/settings/prepare/') servers['initial_server_group'] = gvar['active_group'] groups = gvar['user_groups'] for group in sorted(groups): if group in fetched[host]: continue fetched[host][group] = True host_group_xref['%s::%s' % (host, group)] = server # Switch groups. response = requests(gvar, '/settings/prepare/', {'group': group}) print('Checking: server=%s, group=%s' % (server, group)) response = requests(gvar, '/group/defaults/') _conditionally_restore_a_file(gvar, '%s/groups/%s/defaults' % (host_dir, response['defaults_list'][0]['group_name']), response['defaults_list'][0]) response = requests(gvar, '/group/metadata-list/') for metadata in response['group_metadata_list']: _conditionally_restore_a_file(gvar, '%s/groups/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['metadata_name']), metadata) response = requests(gvar, '/cloud/list/') for cloud in response['cloud_list']: _conditionally_restore_a_file(gvar, '%s/groups/%s/clouds/%s/settings' % (host_dir, cloud['group_name'], cloud['cloud_name']), _get_cloud_settings(cloud)) response = requests(gvar, '/cloud/metadata-list/') for metadata in response['cloud_metadata_list']: _conditionally_restore_a_file(gvar, '%s/groups/%s/clouds/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['cloud_name'], metadata['metadata_name']), metadata) # Restore the server's initial group. response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']}) # Scan remaining backup files building an inventory to restore. remaining_inventory = {} for path in sorted(gvar['backup_path_list']): words = path.split('/') ix = words.index('groups') host = words[ix-1] group = words[ix+1] xref_key = '%s::%s' % (host, group) if xref_key in gvar['host_group_xref']: server = host_group_xref[xref_key] if server not in remaining_inventory: remaining_inventory[server] = {} if group not in remaining_inventory[server]: remaining_inventory[server][group] = [] remaining_inventory[server][group].append(path) else: print('Skipping backup file "%s", you have no access to the server.') continue # Perform remaining conditional restores. for server in sorted(remaining_inventory): host, host_dir = _set_host(gvar, servers, server) # Save the initital server group so it can be restored later. response = requests(gvar, '/settings/prepare/') servers['initial_server_group'] = gvar['active_group'] for group in sorted(remaining_inventory[server]): print('Processing missing files: server=%s, group=%s' % (server, group)) # Switch groups. response = requests(gvar, '/settings/prepare/', {'group': group}) for path in remaining_inventory[server][group]: _conditionally_restore_a_file(gvar, path, None) # Restore the server's initial group. response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']}) _update_git(gvar, 'post-restore') rmtree(gvar['temp_dir'])
def list(gvar): """ List clouds for the active group. """ mandatory = [] required = [] optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-ok', '-r', '-s', '-V', '-VC', '-NV', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/cloud/list/') if response['message']: print(response['message']) # Filter response as requested (or not). cloud_list = _filter_by_cloud_name_and_or_metadata_name(gvar, response['cloud_list']) # Print report. show_active_user_groups(gvar, response) show_table( gvar, cloud_list, [ 'group_name/Group,k', 'cloud_name/Cloud,k', 'enabled/Enabled', 'authurl/URL', 'project_domain_name/Project Domain', 'project/Project', 'user_domain_name/User Domain', 'username/User', 'region/Region', 'spot_price/Spot Price', 'cloud_type/Cloud Type', 'cores_ctl/Control/Cores', 'cores_softmax/SoftMax/Cores', 'cores_max/Max/Cores', 'ram_ctl/Control/RAM', 'ram_max/Max/RAM', 'vm_flavor/Flavor/Cloud Default', 'vm_image/Image/Cloud Default', 'vm_keep_alive/Keep Alive/Cloud Default', 'vm_keyname/Keyname/Cloud Default', 'vm_network/Network/Cloud Default', 'cascading_vm_flavor/Flavor/Cascading Default', 'cascading_vm_image/Image/Cascading Default', 'cascading_vm_keep_alive/Keep Alive/Cascading Default', 'cascading_vm_keyname/Keyname/Cascading Default', 'cascading_vm_network/Network/Cascading Default', 'cacertificate/CA Certificate', 'flavor_exclusions/Flavor Exclusions/Cloud', 'flavor_names/Flavors/Cloud', 'group_exclusions/Group Exclusions/Metadata', 'metadata_names/Filenames/Metadata', ], title="Clouds:", )
def status(gvar): """ List cloud status for the active group. """ mandatory = [] required = [] optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-o', '-ok', '-r', '-s', '-V', '-VC', '-xA'] if gvar['retrieve_options']: return mandatory + required + optional # Check for missing arguments or help required. check_keys(gvar, mandatory, required, optional) # Retrieve data (possibly after changing the group). response = requests(gvar, '/cloud/status/') # Filter response as requested (or not). cloud_status_list = _filter_by_cloud_name_and_or_metadata_name(gvar, response['cloud_status_list']) # Print report show_active_user_groups(gvar, response) show_table( gvar, response['job_status_list'], [ 'group_name/Group,k', 'Jobs', 'Idle', 'Running', 'Completed', 'Other', ], title="Job status:", ) show_table( gvar, cloud_status_list, [ 'group_name/Group,k', 'cloud_name/Cloud,k', 'enabled/Enabled/Defaults', 'default_flavor/Flavor/Defaults', 'default_image/Image/Defaults', 'keep_alive/Keep Alive/Defaults', 'VMs/Total/VMs', 'VMs_starting/Starting/VMs', 'VMs_unregistered/Unregistered/VMs', 'VMs_idle/idle/VMs', 'VMs_running/Running/VMs', 'VMs_retiring/Retiring/VMs', 'VMs_manual/Manual/VMs', 'VMs_in_error/Error/VMs', 'VMs_other/Other/VMs', 'cores_max/Quota/Cores', 'cores_limit/Limit/Cores', 'cores_ctl/Setting/Cores', 'cores_idle/Idle/Cores', 'cores_native/Used/Cores', 'ram_max/Quota/RAM', 'ram_limit/Limit/RAM', 'ram_ctl/Setting/RAM', 'ram_idle/Idle/RAM', 'ram_native/Used/RAM', 'slot_count/Busy/Condor Slots', 'slot_core_count/Busy Cores/Condor Slots', 'slot_idle_core_count/Idle Cores/Condor Slots', 'Foreign_VMs/VMs/Foreign', 'cores_foreign/Cores/Foreign', 'ram_foreign/RAM/Foreign', ], title="Cloud status:", )