def delete(gvar):
"""
Delete settings.
"""
mandatory = ['-s']
required = []
optional = ['-H', '-h', '-Y']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional, requires_server=False)
if os.path.isdir('%s/.csv2/%s' % (gvar['home_dir'], gvar['server'])):
# Confirm settings delete.
if not gvar['user_settings']['yes']:
print(
'Are you sure you want to delete the settings for server "%s"? (yes|..)'
% gvar['server'])
_reply = input()
if _reply != 'yes':
print('%s settings delete "%" cancelled.' % gvar['server'] %
gvar['command_name'])
exit(0)
shutil.rmtree('%s/.csv2/%s' % (gvar['home_dir'], gvar['server']))
else:
print('Error: Settings for server "%s" do not exist.' % gvar['server'])
exit(1)
def metadata_delete(gvar):
"""
Delete a group metadata file.
"""
mandatory = ['-mn']
required = []
optional = ['-g', '-H', '-h', '-s', '-xA', '-Y']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Check that the target groupmetadata file exists.
response = requests(gvar, '/group/list/')
_found = False
for row in response['group_list']:
if row['group_name'] == gvar['active_group']:
metadata_names = row['metadata_names'].split(',')
for metadata_name in metadata_names:
if row['group_name'] == gvar['active_group']:
_found = True
break
if not _found:
print(
'Error: "%s group metadata-delete" cannot delete "%s::%s", file doesn\'t exist.'
% (gvar['command_name'], response['active_group'],
gvar['user_settings']['metadata-name']))
exit(1)
# Confirm group metadata file delete.
if not gvar['user_settings']['yes']:
print(
'Are you sure you want to delete the metadata file "%s::%s"? (yes|..)'
%
(response['active_group'], gvar['user_settings']['metadata-name']))
_reply = input()
if _reply != 'yes':
print('%s group metadata-delete "%s::%s" cancelled.' %
(gvar['command_name'], response['active_group'],
gvar['user_settings']['metadata-name']))
exit(0)
# Delete the group metadata file.
response = requests(gvar,
'/group/metadata-delete/',
form_data={
'metadata_name':
gvar['user_settings']['metadata-name'],
})
if response['message']:
print(response['message'])
def list(gvar):
"""
List settings.
"""
mandatory = []
required = []
optional = ['-H', '-h', '-NV', '-ok', '-r', '-s', '-V', '-VC']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional, requires_server=False)
# Retrive all possible option names ordered by 'server' and then alphabetically.
if os.path.isfile('%s/.csv2/default_server' % gvar['home_dir']):
fd = open('%s/.csv2/default_server' % gvar['home_dir'])
default_server = fd.read()
fd.close
report_title = 'Defaults (default server = %s):' % default_server
else:
report_title = 'Defaults:'
# Retrive all possible option names ordered by 'server' and then alphabetically.
_keys = ['server,k']
for _ix in range(len(gvar['command_keys'])):
key = gvar['command_keys'][_ix][1][2:]
if key not in _keys:
_keys.append(key)
_keys = [_keys[0]] + sorted(_keys[1:])
# Build a queryset of the settings.
_queryset = []
for server in os.listdir('%s/.csv2' % gvar['home_dir']):
if server[0] == '.':
continue
server_path = '%s/.csv2//%s' % (gvar['home_dir'], server)
if os.path.isdir(server_path):
if 'server' not in gvar['command_args'] or server == gvar[
'command_args']['server']:
_fd = open('%s/settings.yaml' % server_path)
_settings = yaml.load(_fd.read())
_fd.close()
for key in sorted(_keys):
if key not in _settings:
if key == 'server':
_settings[key] = server
_queryset.append({'fields': _settings})
# Display results.
show_table(gvar, _queryset, _keys, allow_null=False, title=report_title)
def base_metadata_save(gvar, context=None):
"""
Save the specified cloud or cloud metadata file.
"""
if context is None:
print('Error: This code should not have been called without a context')
traceback.print_tb()
return
if context == 'cloud':
mandatory = ['-cn', '-mn', '-f']
else:
mandatory = ['-mn', '-f']
required = []
optional = ['-F', '-g', '-H', '-h', '-s', '-v', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
if context == 'cloud':
response = requests(gvar,
'/cloud/metadata-fetch/',
query_data={
'cloud_name':
gvar['user_settings']['cloud-name'],
'metadata_name':
gvar['user_settings']['metadata-name']
})
else:
response = requests(gvar,
'/group/metadata-fetch/',
query_data={
'metadata_name':
gvar['user_settings']['metadata-name']
})
if os.path.exists(gvar['user_settings']
['file-path']) and not gvar['user_settings']['force']:
print(
'Error: The specified metadata file "%s" already exists, use -F to overwrite .'
% gvar['user_settings']['file-path'])
exit(1)
# Write the reference copy.
fd = open(gvar['user_settings']['file-path'], 'w')
fd.write(response['metadata'])
fd.close()
def add(gvar):
"""
Add a user.
"""
mandatory = ['-un', '-upw']
required = []
optional = ['-g', '-gn', '-H', '-h', '-SU', '-s', '-ucn', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(
gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
# Create the user.
response = requests(
gvar,
'/user/add/',
form_data
)
if response['message']:
print(response['message'])
def add(gvar):
"""
Add a cloud to the active group.
"""
mandatory = ['-ca', '-cn', '-cp', '-cpw', '-cr', '-ct', '-cu']
required = []
optional = ['-ce', '-cfe', '-cP', '-csp', '-cU', '-g', '-ga', '-gme', '-H', '-h', '-s', '-vc', '-vcs', '-vf', '-vi', '-vk', '-vka', '-vk', '-vn', '-vr', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(
gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
# Create the cloud.
response = requests(
gvar,
'/cloud/add/',
form_data
)
if response['message']:
print(response['message'])
def update(gvar):
"""
Modify a cloud in the active group.
"""
mandatory = ['-cn']
required = []
optional = ['-ca', '-ce', '-cfe', '-cfo', '-cpw', '-cP', '-cp', '-cr', '-csp', '-ct', '-cU', '-cu', '-g', '-ga', '-gme', '-gmo', '-H', '-h', '-s', '-vc', '-vcs', '-vf', '-vi', '-vk', '-vka', '-vk', '-vn', '-vr', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(
gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
if len(form_data) < 2:
print('Error: "%s cloud update" requires at least one option to modify.' % gvar['command_name'])
exit(1)
# Create the cloud.
response = requests(
gvar,
'/cloud/update/',
form_data
)
if response['message']:
print(response['message'])
def update(gvar):
"""
Modify the specified group.
"""
mandatory = ['-gn']
required = []
optional = ['-g', '-gm', '-H', '-h', '-s', '-un', '-uo', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
if len(form_data) < 2:
print(
'Error: "%s group update" requires at least one option to update.'
% gvar['command_name'])
exit(1)
# Create the group.
response = requests(gvar, '/group/update/', form_data)
if response['message']:
print(response['message'])
def metadata_update(gvar):
"""
Update metadata fiel information.
"""
mandatory = ['-mn']
required = []
optional = ['-g', '-H', '-h', '-me', '-mmt', '-mp', '-s', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
if len(form_data) < 2:
print(
'Error: "%s group metadata-update" requires at least one option to modify.'
% gvar['command_name'])
exit(1)
# Create the cloud.
response = requests(gvar, '/group/metadata-update/', form_data)
if response['message']:
print(response['message'])
def delete(gvar):
"""
Delete a cloud from the active group.
"""
mandatory = ['-cn']
required = []
optional = ['-g', '-H', '-h', '-s', '-xA', '-Y']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Check that the target cloud exists.
response = requests(gvar, '/cloud/list/')
_found = False
for row in response['cloud_list']:
if row['cloud_name'] == gvar['user_settings']['cloud-name']:
_found = True
break
if not _found:
print('Error: "%s cloud delete" cannot delete "%s", cloud doesn\'t exist in group "%s".' % (gvar['command_name'], gvar['user_settings']['cloud-name'], response['active_group']))
exit(1)
# Confirm cloud delete.
if not gvar['user_settings']['yes']:
print('Are you sure you want to delete cloud "%s::%s"? (yes|..)' % (response['active_group'], gvar['user_settings']['cloud-name']))
_reply = input()
if _reply != 'yes':
print('%s cloud delete "%s::%s" cancelled.' % (gvar['command_name'], response['active_group'], gvar['user_settings']['cloud-name']))
exit(0)
# Delete the cloud.
response = requests(
gvar,
'/cloud/delete/',
form_data = {
'cloud_name': gvar['user_settings']['cloud-name']
}
)
if response['message']:
print(response['message'])
def delete(gvar):
"""
Delete a user.
"""
mandatory = ['-un']
required = []
optional = ['-g', '-H', '-h', '-s', '-xA', '-Y']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Check that the target user exists.
response = requests(gvar, '/user/list/')
_found = False
for row in response['user_list']:
if row['username'] == gvar['user_settings']['username']:
_found = True
break
if not _found:
print('Error: "%s user delete" cannot delete "%s", user doesn\'t exist.' % (gvar['command_name'], gvar['user_settings']['username']))
exit(1)
# Confirm user delete.
if not gvar['user_settings']['yes']:
print('Are you sure you want to delete user "%s"? (yes|..)' % gvar['user_settings']['username'])
_reply = input()
if _reply != 'yes':
print('%s user delete "%s" cancelled.' % (gvar['command_name'], gvar['user_settings']['username']))
exit(0)
# Delete the user.
response = requests(
gvar,
'/user/delete/',
form_data = {
'username': gvar['user_settings']['username']
}
)
if response['message']:
print(response['message'])
def set(gvar):
"""
Modify settings.
"""
mandatory = ['-s']
required = []
optional = ['*']
not_optional = ['-br']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar,
mandatory,
required,
optional,
not_optional=not_optional,
requires_server=False)
# If this is the user's first server, make it the default.
if len(os.listdir('%s/.csv2' % gvar['home_dir'])) < 1:
gvar['set_default_server'] = True
# Make the server directory, if necessary.
if not os.path.exists('%s/.csv2/%s' % (gvar['home_dir'], gvar['server'])):
os.makedirs('%s/.csv2/%s' % (gvar['home_dir'], gvar['server']),
mode=0o700)
# Write the default server file.
if gvar['set_default_server']:
_fd = open('%s/.csv2/default_server' % gvar['home_dir'], 'w')
_fd.write(gvar['server'])
_fd.close()
os.chmod('%s/.csv2/default_server' % gvar['home_dir'], 0o600)
# Write the settings file.
_fd = open(
'%s/.csv2/%s/settings.yaml' % (gvar['home_dir'], gvar['server']), 'w')
_fd.write(yaml.dump(gvar['user_settings']))
_fd.close()
os.chmod('%s/.csv2/%s/settings.yaml' % (gvar['home_dir'], gvar['server']),
0o600)
def metadata_list(gvar):
"""
List clouds for the active group.
"""
mandatory = []
required = []
optional = [
'-CSEP', '-CSV', '-g', '-H', '-h', '-mn', '-NV', '-ok', '-r', '-s',
'-V', '-VC', '-xA'
]
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/group/metadata-list/')
if response['message']:
print(response['message'])
# Filter response as requested (or not).
group_metadata_list = _filter_by_group_name_and_or_metadata_name(
gvar, response['group_metadata_list'])
# Print report.
show_active_user_groups(gvar, response)
show_table(
gvar,
group_metadata_list,
[
'group_name/Group,k',
'metadata_name/Metadata Filename,k',
'enabled/Enabled',
'priority/Priority',
'mime_type/MIME Type',
],
title="Active Group/Metadata:",
)
def list(gvar):
"""
List groups.
"""
mandatory = []
required = []
optional = [
'-CSEP', '-CSV', '-g', '-gn', '-H', '-h', '-NV', '-ok', '-r', '-s',
'-V', '-VC', '-xA'
]
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/group/list/')
if response['message']:
print(response['message'])
# Filter response as requested (or not).
group_list = _filter_by_group_name_and_or_metadata_name(
gvar, response['group_list'])
# Print report
show_active_user_groups(gvar, response)
show_table(
gvar,
group_list,
[
'group_name/Group,k',
'condor_central_manager/Central Manager',
'metadata_names/Metadata Filenames',
],
title="Groups:",
)
def list(gvar):
"""
List users.
"""
mandatory = []
required = []
optional = ['-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-ok', '-r', '-s', '-un', '-V', '-VC', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the user).
response = requests(gvar, '/user/list/')
if response['message']:
print(response['message'])
# Filter response as requested (or not).
user_list = _filter_by_user(gvar, response['user_list'])
# Print report
show_active_user_groups(gvar, response)
show_table(
gvar,
user_list,
[
'username/Username,k',
'cert_cn/Common Name',
'active_group/Active Group',
'user_groups/User Groups',
'available_groups/Not In Groups',
'is_superuser/Super User',
'join_date/Joined',
],
title="Users:",
)
def defaults(gvar):
"""
Modify the specified group defaults.
"""
mandatory = []
required = []
optional = [
'-CSEP', '-CSV', '-g', '-H', '-h', '-htcf', '-htcn', '-htcu', '-jc',
'-jd', '-jr', '-js', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-vf',
'-vi', '-vka', '-vk', '-vn', '-xA'
]
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
# List the current defaults. If the form_data contains any optional fields,
# those values will be updated before the list is retrieved.
response = requests(gvar, '/group/defaults/', form_data)
if response['message']:
print(response['message'])
# Print report
show_active_user_groups(gvar, response)
show_table(
gvar,
response['defaults_list'],
[
'group_name/Group,k',
'htcondor_fqdn/FQDN/HTCondor',
'htcondor_name/Name/HTCondor',
'htcondor_other_submitters/Users/HTCondor',
'vm_flavor/Flavor/VM',
'vm_image/Image/VM',
'vm_keep_alive/Keep Alive/VM',
'vm_keyname/Keyname/VM',
'vm_network/Network/VM',
'job_cpus/Cores/Job',
'job_disk/Disk (GBs)/Job',
'job_ram/RAM (MBs)/Job',
'job_swap/Swap (GBs)/Job',
],
title="Active Group Defaults:",
)
def config(gvar):
"""
Modify the specified group defaults.
"""
mandatory = []
required = []
optional = [
'-c', '-cc', '-CSEP', '-CSV', '-dci', '-djg', '-eg', '-g', '-H', '-h',
'-lf', '-ll', '-nld', '-NV', '-ok', '-r', '-s', '-sic', '-siC', '-sif',
'-sii', '-sij', '-sik', '-sil', '-sim', '-sin', '-siv', '-sml', '-sms',
'-V', '-VC', '-xA'
]
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
# List the current defaults. If the form_data contains any optional fields,
# those values will be updated before the list is retrieved.
response = requests(gvar, '/server/config/', form_data)
if response['message']:
print(response['message'])
# Print report
show_active_user_groups(gvar, response)
show_table(
gvar,
response['config_list'],
[
'category/Category,k',
'config_key/Config Key,k',
'config_type/Type',
'config_value/Value',
],
title="Server Configuration:",
)
def metadata_load(gvar):
"""
Load a new group metadata file.
"""
mandatory = ['-f', '-mn']
required = []
optional = ['-g', '-H', '-h', '-me', '-mmt', '-mp', '-s', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
if not os.path.exists(gvar['user_settings']['file-path']):
print('Error: The specified metadata file "%s" does not exist.' %
gvar['user_settings']['file-path'])
exit(1)
# # Verify the changed metadata file and build input form data.
# form_data = {
# **verify_yaml_file(gvar['user_settings']['file-path']),
# 'metadata_name': gvar['user_settings']['metadata-name'],
# }
# Replace the metadata file.
response = requests(gvar, '/group/metadata-add/', {
**form_data,
**verify_yaml_file(gvar['user_settings']['file-path']),
})
if response['message']:
print(response['message'])
def settings(gvar):
"""
Modify the specified user.
"""
mandatory = []
required = []
optional = ['-H', '-h', '-s', '-upw', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
if len(form_data) < 1:
print(
'Error: "%s my settings" requires at least one option to update.' %
gvar['command_name'])
exit(1)
# Create the user.
response = requests(gvar, '/user/settings/', form_data)
if response['message']:
print(response['message'])
if 'server' not in gvar['command_args']:
gvar['command_args']['server'] = gvar['server']
gvar['user_settings']['server-password'] = gvar['user_settings'][
'user-password']
del gvar['user_settings']['user-password']
csv2_defaults.set(gvar)
def update(gvar):
"""
Modify a VM in the active group.
"""
mandatory = ['-vh', '-vo']
required = []
optional = ['-cn', '-g', '-H', '-h', '-s', '-vS', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
form_data = check_keys(gvar,
mandatory,
required,
optional,
key_map=KEY_MAP)
# Create the cloud.
response = requests(gvar, '/vm/update/', form_data)
if response['message']:
print(response['message'])
def list(gvar):
"""
List VMs for the active group.
"""
mandatory = []
required = []
optional = [
'-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-ok', '-r', '-s',
'-V', '-VC', '-vc', '-vd', '-vF', '-vf', '-vh', '-vr', '-vS', '-vs',
'-xA'
]
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/vm/list/%s' % _selector(gvar))
if response['message']:
print(response['message'])
# Filter response as requested (or not).
vm_list = _filter(gvar, response['vm_list'])
# Print report.
show_active_user_groups(gvar, response)
show_table(
gvar,
vm_list,
[
'group_name/Group,k',
'cloud_name/Cloud,k',
'hostname/Hostname,k',
'vmid/VMID',
'vm_ips/IPs',
'vm_floating_ips/Floating IPs',
'auth_url/Authorization URL',
'project/Project',
'status/Status',
'flavor_id/Flavor ID',
'task/Task',
'power_status/Power Status',
'start_time/Start Time',
'retire_request_time/Retire Request Time',
'retired_time/Retired Time',
'terminate/Terminate',
'terminate_time/Terminate Time',
'status_changed_time/Status Change Time',
'last_updated/Last Updated',
'flavor_name/Flavor',
'condor_slots/Condor Slots',
'foreign_vm/Foreign',
'cores/cores',
'disk/Disk (GBs)',
'ram/Ram (MBs)',
'swap/Swap (GBs)',
'poller_status/Poller Status',
'manual_control/Manual_Control',
],
title="VMs:",
)
def metadata_edit(gvar):
"""
Edit the specified group metadata file.
"""
mandatory = ['-mn']
required = ['-te']
optional = ['-g', '-H', '-h', '-s', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(
gvar,
'/group/metadata-fetch/%s/' % gvar['user_settings']['metadata-name'])
# Ensure the fetch directory structure exists.
fetch_dir = '%s/.csv2/%s/files/%s/metadata' % (
gvar['home_dir'],
gvar['server'],
response['group_name'],
)
if not os.path.exists(fetch_dir):
os.makedirs(fetch_dir, mode=0o700)
# Write the reference copy.
fd = open('%s/.%s' % (fetch_dir, response['metadata_name']), 'w')
# fd.write('# metadata_enabled: %s, metadata_mime_type: %s\n%s' % (response['metadata_enabled'], response['metadata_mime_type'], response['metadata']))
fd.write(response['metadata'])
fd.close()
# Write the edit copy.
fd = open('%s/%s' % (fetch_dir, response['metadata_name']), 'w')
# fd.write('# metadata_enabled: %s, metadata_mime_type: %s\n%s' % (response['metadata_enabled'], response['metadata_mime_type'], response['metadata']))
fd.write(response['metadata'])
fd.close()
# Edit the metadata file.
editor = ''
if 'text-editor' in gvar['user_settings']:
editor = gvar['user_settings']['text-editor']
else:
editor = os.getenv('EDITOR')
p = Popen([editor, '%s/%s' % (fetch_dir, response['metadata_name'])])
p.communicate()
if filecmp.cmp('%s/.%s' % (fetch_dir, response['metadata_name']),
'%s/%s' % (fetch_dir, response['metadata_name'])):
print('%s group metadata-edit "%s::%s" completed, no changes.' %
(gvar['command_name'], response['group_name'],
response['metadata_name']))
exit(0)
# Verify the changed metadata file.
form_data = {
**verify_yaml_file('%s/%s' % (fetch_dir, response['metadata_name'])),
'metadata_name':
response['metadata_name'],
}
# Replace the metadata file.
response = requests(gvar, '/group/metadata-update/', form_data)
if response['message']:
print(response['message'])
def backup(gvar):
"""
Backup all user data for all groups/clouds for each server configured in the user defaults.
"""
mandatory = []
required = ['-br']
optional = ['-bk', '-xA']
servers = {}
if gvar['retrieve_options']:
return mandatory + required + optional
# Retrieve the backup repository and all server information.
servers['settings'], servers['xref'] = _get_repository_and_servers(gvar)
# Check for missing arguments or help required.
check_keys(
gvar,
mandatory,
required,
optional)
# If the backup directory is an encrypted git repository, create a working temorary directory.
_update_git(gvar, 'pre-backup')
# If the backup directory is an encrypted git repository, create a working temorary directory.
if 'backup-key' in gvar['user_settings'] and os.path.exists('%s/.git' % gvar['user_settings']['backup-repository']):
gvar['temp_dir'] = mkdtemp()
else:
gvar['temp_dir'] = None
# Retrieve data to backup for each cloudscheduler server.
fetched = {}
for server in sorted(servers['settings']):
host, host_dir = _set_host(gvar, servers, server)
if not host:
print('Skipping server="%s", no server address.' % server)
continue
if host not in fetched:
fetched[host] = {}
# Save the initital server group so it can be restored later.
response = requests(gvar, '/settings/prepare/')
servers['initial_server_group'] = gvar['active_group']
groups = gvar['user_groups']
for group in sorted(groups):
if group in fetched[host]:
continue
fetched[host][group] = True
# Switch groups.
response = requests(gvar, '/settings/prepare/', {'group': group})
print('Fetching: server=%s, group=%s' % (server, group))
response = requests(gvar, '/group/defaults/')
_create_backup_file(gvar, '%s/groups/%s/defaults' % (host_dir, response['defaults_list'][0]['group_name']), response['defaults_list'][0])
response = requests(gvar, '/group/metadata-list/')
for metadata in response['group_metadata_list']:
_create_backup_file(gvar, '%s/groups/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['metadata_name']), metadata)
response = requests(gvar, '/cloud/list/')
for cloud in response['cloud_list']:
_create_backup_file(gvar, '%s/groups/%s/clouds/%s/settings' % (host_dir, cloud['group_name'], cloud['cloud_name']), _get_cloud_settings(cloud))
response = requests(gvar, '/cloud/metadata-list/')
for metadata in response['cloud_metadata_list']:
_create_backup_file(gvar, '%s/groups/%s/clouds/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['cloud_name'], metadata['metadata_name']), metadata)
# Restore the server's initial group.
response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']})
_update_git(gvar, 'post-backup')
if gvar['temp_dir']:
rmtree(gvar['temp_dir'])
def restore(gvar):
"""
Restore user data.
"""
mandatory = []
required = ['-br']
optional = ['-bk', '-xA']
servers = {}
if gvar['retrieve_options']:
return mandatory + required + optional
# Retrieve the backup repository and all server information.
servers['settings'], servers['xref'] = _get_repository_and_servers(gvar)
# Check for missing arguments or help required.
check_keys(
gvar,
mandatory,
required,
optional)
# Commit any outstanding repository updates prior to the restore.
_update_git(gvar, 'pre-restore')
# Retrieve an inventory of the current backup files.
gvar['backup_path_list'] = {}
_get_backup_path_list(gvar, gvar['user_settings']['backup-repository'])
# We need a temporary working directory.
gvar['temp_dir'] = mkdtemp()
# Retrieve data from each cloudscheduler server to compare with its' backup. If the
# backup is different, conditionally (allow the user to choose) restore it.
fetched = {}
host_group_xref = {}
for server in sorted(servers['settings']):
host, host_dir = _set_host(gvar, servers, server)
if not host:
print('Skipping server="%s", no server address.' % server)
continue
if host not in fetched:
fetched[host] = {}
# Save the initital server group so it can be restored later.
response = requests(gvar, '/settings/prepare/')
servers['initial_server_group'] = gvar['active_group']
groups = gvar['user_groups']
for group in sorted(groups):
if group in fetched[host]:
continue
fetched[host][group] = True
host_group_xref['%s::%s' % (host, group)] = server
# Switch groups.
response = requests(gvar, '/settings/prepare/', {'group': group})
print('Checking: server=%s, group=%s' % (server, group))
response = requests(gvar, '/group/defaults/')
_conditionally_restore_a_file(gvar, '%s/groups/%s/defaults' % (host_dir, response['defaults_list'][0]['group_name']), response['defaults_list'][0])
response = requests(gvar, '/group/metadata-list/')
for metadata in response['group_metadata_list']:
_conditionally_restore_a_file(gvar, '%s/groups/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['metadata_name']), metadata)
response = requests(gvar, '/cloud/list/')
for cloud in response['cloud_list']:
_conditionally_restore_a_file(gvar, '%s/groups/%s/clouds/%s/settings' % (host_dir, cloud['group_name'], cloud['cloud_name']), _get_cloud_settings(cloud))
response = requests(gvar, '/cloud/metadata-list/')
for metadata in response['cloud_metadata_list']:
_conditionally_restore_a_file(gvar, '%s/groups/%s/clouds/%s/metadata/%s' % (host_dir, metadata['group_name'], metadata['cloud_name'], metadata['metadata_name']), metadata)
# Restore the server's initial group.
response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']})
# Scan remaining backup files building an inventory to restore.
remaining_inventory = {}
for path in sorted(gvar['backup_path_list']):
words = path.split('/')
ix = words.index('groups')
host = words[ix-1]
group = words[ix+1]
xref_key = '%s::%s' % (host, group)
if xref_key in gvar['host_group_xref']:
server = host_group_xref[xref_key]
if server not in remaining_inventory:
remaining_inventory[server] = {}
if group not in remaining_inventory[server]:
remaining_inventory[server][group] = []
remaining_inventory[server][group].append(path)
else:
print('Skipping backup file "%s", you have no access to the server.')
continue
# Perform remaining conditional restores.
for server in sorted(remaining_inventory):
host, host_dir = _set_host(gvar, servers, server)
# Save the initital server group so it can be restored later.
response = requests(gvar, '/settings/prepare/')
servers['initial_server_group'] = gvar['active_group']
for group in sorted(remaining_inventory[server]):
print('Processing missing files: server=%s, group=%s' % (server, group))
# Switch groups.
response = requests(gvar, '/settings/prepare/', {'group': group})
for path in remaining_inventory[server][group]:
_conditionally_restore_a_file(gvar, path, None)
# Restore the server's initial group.
response = requests(gvar, '/settings/prepare/', {'group': servers['initial_server_group']})
_update_git(gvar, 'post-restore')
rmtree(gvar['temp_dir'])
def list(gvar):
"""
List clouds for the active group.
"""
mandatory = []
required = []
optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-ok', '-r', '-s', '-V', '-VC', '-NV', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/cloud/list/')
if response['message']:
print(response['message'])
# Filter response as requested (or not).
cloud_list = _filter_by_cloud_name_and_or_metadata_name(gvar, response['cloud_list'])
# Print report.
show_active_user_groups(gvar, response)
show_table(
gvar,
cloud_list,
[
'group_name/Group,k',
'cloud_name/Cloud,k',
'enabled/Enabled',
'authurl/URL',
'project_domain_name/Project Domain',
'project/Project',
'user_domain_name/User Domain',
'username/User',
'region/Region',
'spot_price/Spot Price',
'cloud_type/Cloud Type',
'cores_ctl/Control/Cores',
'cores_softmax/SoftMax/Cores',
'cores_max/Max/Cores',
'ram_ctl/Control/RAM',
'ram_max/Max/RAM',
'vm_flavor/Flavor/Cloud Default',
'vm_image/Image/Cloud Default',
'vm_keep_alive/Keep Alive/Cloud Default',
'vm_keyname/Keyname/Cloud Default',
'vm_network/Network/Cloud Default',
'cascading_vm_flavor/Flavor/Cascading Default',
'cascading_vm_image/Image/Cascading Default',
'cascading_vm_keep_alive/Keep Alive/Cascading Default',
'cascading_vm_keyname/Keyname/Cascading Default',
'cascading_vm_network/Network/Cascading Default',
'cacertificate/CA Certificate',
'flavor_exclusions/Flavor Exclusions/Cloud',
'flavor_names/Flavors/Cloud',
'group_exclusions/Group Exclusions/Metadata',
'metadata_names/Filenames/Metadata',
],
title="Clouds:",
)
def status(gvar):
"""
List cloud status for the active group.
"""
mandatory = []
required = []
optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-NV', '-o', '-ok', '-r', '-s', '-V', '-VC', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(gvar, mandatory, required, optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/cloud/status/')
# Filter response as requested (or not).
cloud_status_list = _filter_by_cloud_name_and_or_metadata_name(gvar, response['cloud_status_list'])
# Print report
show_active_user_groups(gvar, response)
show_table(
gvar,
response['job_status_list'],
[
'group_name/Group,k',
'Jobs',
'Idle',
'Running',
'Completed',
'Other',
],
title="Job status:",
)
show_table(
gvar,
cloud_status_list,
[
'group_name/Group,k',
'cloud_name/Cloud,k',
'enabled/Enabled/Defaults',
'default_flavor/Flavor/Defaults',
'default_image/Image/Defaults',
'keep_alive/Keep Alive/Defaults',
'VMs/Total/VMs',
'VMs_starting/Starting/VMs',
'VMs_unregistered/Unregistered/VMs',
'VMs_idle/idle/VMs',
'VMs_running/Running/VMs',
'VMs_retiring/Retiring/VMs',
'VMs_manual/Manual/VMs',
'VMs_in_error/Error/VMs',
'VMs_other/Other/VMs',
'cores_max/Quota/Cores',
'cores_limit/Limit/Cores',
'cores_ctl/Setting/Cores',
'cores_idle/Idle/Cores',
'cores_native/Used/Cores',
'ram_max/Quota/RAM',
'ram_limit/Limit/RAM',
'ram_ctl/Setting/RAM',
'ram_idle/Idle/RAM',
'ram_native/Used/RAM',
'slot_count/Busy/Condor Slots',
'slot_core_count/Busy Cores/Condor Slots',
'slot_idle_core_count/Idle Cores/Condor Slots',
'Foreign_VMs/VMs/Foreign',
'cores_foreign/Cores/Foreign',
'ram_foreign/RAM/Foreign',
],
title="Cloud status:",
)
def list(gvar):
"""
List clouds for the active group.
"""
mandatory = []
required = []
optional = ['-cn', '-CSEP', '-CSV', '-g', '-H', '-h', '-jh', '-jI', '-ji', '-jp', '-jR', '-jrc', '-jrd', '-jrr', '-jrs', '-jS', '-jtc', '-ju', '-NV', '-ok', '-r', '-s', '-V', '-VC', '-xA']
if gvar['retrieve_options']:
return mandatory + required + optional
# Check for missing arguments or help required.
check_keys(
gvar,
mandatory,
required,
optional)
# Retrieve data (possibly after changing the group).
response = requests(gvar, '/job/list/')
if response['message']:
print(response['message'])
# Filter response as requested (or not).
job_list = _filter(gvar, response['job_list'])
# Print report.
show_active_user_groups(gvar, response)
show_table(
gvar,
job_list,
[
'group_name/Group,k',
'global_job_id/Job ID,k',
'cluster_id/Cluster ID',
'proc_id/Process ID',
'user/User',
'user_data/User Data',
'requirements/Requirements',
'target_clouds/Target Clouds',
'cloud_name/Cloud',
'instance_type/Instance Type',
'request_cpus/CPUs/Requested',
'request_ram/RAM {MBs}/Requested',
'request_disk/Disk {GBs}/Requested',
'request_swap/Swap (GBs)/Requested',
'job_per_core/Jobs per Core',
'image/Image',
'network/Network',
'job_priority/Priority/Job',
'job_status/Status Code/Job',
'js_idle/Idle/Job Status Flags',
'js_running/Running/Job Status Flags',
'js_completed/Completed/Job Status Flags',
'js_held/Held/Job Status Flags',
'js_other/Other/Job Status Flags',
'keep_alive/Keep Alive (seconds)',
'max_price/Max Spot Price',
'entered_current_status/State Change Date',
'q_date/Queued Date',
'held_reason/Held Job Reason',
],
title="Jobs",
)
