def memstore(reservation,
key_path=None,
admin_id=None,
connectionwrapper=None,
mountpoint_path=start_defaults.mountpoint_path(),
silent=False):
'''Stop a running RADOS-Ceph cluster using memstore.
Args:
reservation (`metareserve.Reservation`): Reservation object with all nodes to start RADOS-Ceph on.
key_path (str): Path to SSH key, which we use to connect to nodes. If `None`, we do not authenticate using an IdentityFile.
admin_id (optional int): Node id of the ceph admin. If `None`, the node with lowest public ip value (string comparison) will be picked.
connectionwrapper (optional RemotoSSHWrapper): If set, uses given connection, instead of building a new one.
mountpoint_path (optional str): Path where CephFS is mounted on all nodes.
silent (optional bool): If set, we only print errors and critical info. Otherwise, more verbose output.
retries (optional int): Number of tries we try to perform potentially-crashing operations.
Returns:
`True` on success, `False` otherwise.'''
if (not reservation) or len(reservation) == 0:
raise ValueError('Reservation does not contain any items' +
(' (reservation=None)' if not reservation else ''))
admin_picked, _ = _pick_admin(reservation, admin=admin_id)
print('Picked admin node: {}'.format(admin_picked))
local_connections = connectionwrapper == None
if local_connections:
ssh_kwargs = {
'IdentitiesOnly': 'yes',
'User': admin_picked.extra_info['user'],
'StrictHostKeyChecking': 'no'
}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrapper = get_wrapper(admin_picked,
admin_picked.ip_public,
silent=silent,
ssh_params=ssh_kwargs)
rados_module = _generate_module_stop()
state_ok = _stop_rados(connectionwrapper.connection,
rados_module,
reservation,
mountpoint_path,
silent=silent)
if local_connections:
close_wrappers([connectionwrapper])
if state_ok:
prints('Stopping RADOS-Ceph succeeded.')
return True
else:
printe('Stopping RADOS-Ceph failed on some nodes.')
return False
def execute(reservation, key_path, paths, dest, silent, copy_multiplier, link_multiplier, *args, **kwargs):
'''Deploy data on remote RADOS-Ceph clusters, on an existing reservation.
Dataset sizes can be inflated on the remote, using 2 strategies:
1. link multiplication: Every dataset file receives `x` hardlinks.
The hardlinks ensure the dataset size appears to be `x` times larger, but in reality, just the original file consumes space.
This method is very fast, but has drawbacks: Only the original files are stored by Ceph.
When using the RADOS-Arrow connector, this means Arrow will spam only the nodes that contain the original data.
E.g: If we deploy 1 file of 64MB, with link multiplier 1024, the data will apear to be 64GB.
The storage space used on RADOS-Ceph will still be 64MB, because we have 1 real file of 64MB, and 1023 hardlinks to that 1 file.
The actual data is only stored on 3 OSDs (with default Ceph Striping factor 3).
Now, Arrow will spam all work to those 3 OSDs containing the data, while the rest is idle.
2. file multiplication: Every dataset file receives `x` copies.
This method is slower than the one listed above, because real data has to be copied.
It also actually increases storage usage, contrary to above.
However, because we multiply real data, the load is guaranteed to be balanced across nodes, as far as Ceph does that.
Note that mutiple multiplication techniques can be combined, in which case they stack.
E.g: If we deploy 1 file of 64MB, with a copy multiplier 4 and a link multiplier 1024, we get 4 real files (1 original + 3 copies),
and each file gets 1023 hardlinks assigned to it.
Returns:
`True` on success, `False` otherwise.'''
connectionwrapper = kwargs.get('connectionwrapper')
admin_id = kwargs.get('admin_id')
stripe = kwargs.get('stripe') or defaults.stripe()
if stripe < 4:
raise ValueError('Stripe size must be equal to or greater than 4MB (and a multiple of 4MB)!')
if stripe % 4 != 0:
raise ValueError('Stripe size must be a multiple of 4MB!')
admin_node, _ = _pick_admin(reservation, admin=admin_id)
use_local_connections = connectionwrapper == None
if use_local_connections: # We did not get any connections, so we must make them
ssh_kwargs = {'IdentitiesOnly': 'yes', 'StrictHostKeyChecking': 'no'}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrapper = ssh_wrapper.get_wrapper(admin_node, admin_node.ip_public, ssh_params=_merge_kwargs(ssh_kwargs, {'User': admin_node.extra_info['user']}), silent=silent)
else: # We received connections, need to check if they are valid.
if not connectionwrapper.open:
raise ValueError('Provided connection is not open.')
retval = _execute_internal(connectionwrapper, reservation, paths, dest, silent, copy_multiplier, link_multiplier, admin_node, stripe)
if not use_local_connections:
ssh_wrapper.close_wrappers([connectionwrapper])
return retval
def clean(reservation,
paths,
key_path=None,
admin_id=None,
connectionwrapper=None,
mountpoint_path=start_defaults.mountpoint_path(),
silent=False):
'''Cleans data from the RADOS-Ceph cluster, on an existing reservation.
Args:
reservation (`metareserve.Reservation`): Reservation object with all nodes to start RADOS-Ceph on.
paths (list(str)): Data paths to delete to the remote cluster. Mountpoint path is always prepended.
key_path (optional str): Path to SSH key, which we use to connect to nodes. If `None`, we do not authenticate using an IdentityFile.
admin_id (optional int): Node id of the ceph admin. If `None`, the node with lowest public ip value (string comparison) will be picked.
mountpoint_path (optional str): Path where CephFS is mounted on all nodes.
silent (optional bool): If set, we only print errors and critical info. Otherwise, more verbose output.
Returns:
`True` on success, `False` otherwise.'''
if (not reservation) or len(reservation) == 0:
raise ValueError('Reservation does not contain any items' +
(' (reservation=None)' if not reservation else ''))
admin_picked, _ = _pick_admin(reservation, admin=admin_id)
print('Picked admin node: {}'.format(admin_picked))
local_connections = connectionwrapper == None
if local_connections:
ssh_kwargs = {
'IdentitiesOnly': 'yes',
'User': admin_picked.extra_info['user'],
'StrictHostKeyChecking': 'no'
}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrapper = get_wrapper(admin_picked.ip_public,
silent=True,
ssh_params=ssh_kwargs)
if not any(paths):
_, _, exitcode = remoto.process.check(
connectionwrapper.connection,
'sudo rm -rf {}/*'.format(mountpoint_path),
shell=True)
state_ok = exitcode == 0
else:
paths = [x if x[0] != '/' else x[1:] for x in paths]
with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count() -
1) as executor:
if not silent:
print('Deleting data...')
futures_rm = [
executor.submit(remoto.process.check,
connectionwrapper.connection,
'sudo rm -rf {}'.format(
fs.join(mountpoint_path, path)),
shell=True) for path in paths
]
state_ok = all(x.result()[2] == 0 for x in futures_rm)
if state_ok:
prints('Data deleted.')
else:
printe('Could not delete data.')
if local_connections:
close_wrappers([connectionwrapper])
return state_ok
def bluestore(reservation,
key_path=None,
admin_id=None,
connectionwrapper=None,
mountpoint_path=defaults.mountpoint_path(),
osd_op_threads=defaults.osd_op_threads(),
osd_pool_size=defaults.osd_pool_size(),
osd_max_obj_size=defaults.osd_max_obj_size(),
placement_groups=None,
use_client_cache=True,
device_path=None,
silent=False,
retries=defaults.retries()):
'''Boot RADOS-Ceph on an existing reservation, running bluestore.
Requires either a "device_path" key to be set in the extra info of all OSD nodes, or the "device_path" parameter must be set.
Should point to device to use with bluestore on all nodes.
Args:
reservation (metareserve.Reservation): Reservation object with all nodes to start RADOS-Ceph on.
key_path (optional str): Path to SSH key, which we use to connect to nodes. If `None`, we do not authenticate using an IdentityFile.
admin_id (optional int): Node id of the ceph admin. If `None`, the node with lowest public ip value (string comparison) will be picked.
connectionwrapper (optional RemotoSSHWrapper): If set, uses given connection, instead of building a new one.
mountpoint_path (optional str): Path where CephFS will be mounted on all nodes.
osd_op_threads (optional int): Number of op threads to use for each OSD. Make sure this number is not greater than the amount of cores each OSD has.
osd_pool_size (optional int): Fragmentation of object to given number of OSDs. Must be less than or equal to amount of OSDs.
osd_max_obj_size (int): Maximal object size in bytes. Normal=128*1024*1024 (128MB).
placement_groups (optional int): Amount of placement groups in Ceph. If not set, we use the recommended formula `(num osds * 100) / (pool size`, as found here: https://ceph.io/pgcalc/.
use_client_cache (bool): Toggles using cephFS I/O cache.
device_path (optional str): If set, overrides the "device_path" extra info for all nodes with given value. Should point to device to use with bluestore on all nodes.
silent (optional bool): If set, we only print errors and critical info. Otherwise, more verbose output.
retries (optional int): Number of tries we try to perform potentially-crashing operations.
Returns:
`(True, admin_node_id)` on success, `(False, None)` otherwise.'''
if not reservation or len(reservation) == 0:
raise ValueError('Reservation does not contain any items' +
(' (reservation=None)' if not reservation else ''))
if isinstance(placement_groups, int):
if placement_groups < 1:
raise ValueError(
'Amount of placement groups must be higher than zero!')
else: # We assume `placememt_groups = None`
placement_groups = _internal_compute_placement_groups(
reservation=reservation)
if device_path: # We got an overriding device_path value
for x in reservation.nodes:
if 'designations' in x.extra_info and Designation.OSD.name.lower(
) in x.extra_info['designations'].split(','):
x.extra_info['device_path'] = device_path
else:
if any(True for x in reservation.nodes
if 'designations' in x.extra_info and Designation.OSD.name.
lower() in x.extra_info['designations'].split(',')
and not 'device_path' in x.extra_info
): # We lack at least 1 device_path value.
printe(
'Missing "device_path" specifier on the following nodes:\n{}'.
format('\n'.join(
'\t{}'.format(x) for x in reservation.nodes
if 'designations' in x.extra_info and Designation.OSD.name.
lower() in x.extra_info['designations'].split(',')
and not 'device_path' in x.extra_info)))
return False, None
admin_picked, _ = _internal_pick_admin(reservation, admin=admin_id)
printc('Picked admin node: {}'.format(admin_picked), Color.CAN)
local_connections = connectionwrapper == None
if local_connections:
ssh_kwargs = {
'IdentitiesOnly': 'yes',
'User': admin_picked.extra_info['user'],
'StrictHostKeyChecking': 'no'
}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrapper = get_wrapper(admin_picked,
admin_picked.ip_public,
silent=silent,
ssh_params=ssh_kwargs)
rados_module = _generate_module_start()
state_ok = _start_rados(connectionwrapper.connection,
rados_module,
reservation,
mountpoint_path,
osd_op_threads,
osd_pool_size,
osd_max_obj_size,
placement_groups,
use_client_cache,
silent=silent,
retries=retries)
if local_connections:
close_wrappers([connectionwrapper])
if state_ok:
prints('Started RADOS-Ceph succeeded.')
return True, admin_picked.node_id
else:
printe('Starting RADOS-Ceph failed on some nodes.')
return False, None
def install(reservation,
install_dir=defaults.install_dir(),
key_path=None,
admin_id=None,
connectionwrapper=None,
arrow_url=defaults.arrow_url(),
use_sudo=defaults.use_sudo(),
force_reinstall=False,
debug=False,
silent=False,
cores=defaults.cores()):
'''Installs RADOS-ceph on remote cluster.
Warning: Requires that usernames on remote cluster nodes are equivalent.
Warning: Requires passwordless communication between nodes on the local network. Use "install_ssh()" to accomplish this.
Args:
reservation (`metareserve.Reservation`): Reservation object with all nodes to install RADOS-Ceph on.
install_dir (optional str): Location on remote host to compile RADOS-arrow in.
key_path (optional str): Path to SSH key, which we use to connect to nodes. If `None`, we do not authenticate using an IdentityFile.
admin_id (optional int): Node id that must become the admin. If `None`, the node with lowest public ip value (string comparison) will be picked.
connectionwrapper (optional RemotoSSHWrapper): If set, uses given connection, instead of building a new one.
arrow_url (optional str): Download URL for Arrow library to use with RADOS-Ceph.
use_sudo (optional bool): If set, uses sudo during installation. Tries to avoid it otherwise.
force_reinstall (optional bool): If set, we always will re-download and install Arrow. Otherwise, we will skip installing if we already have installed Arrow.
debug (optional bool): If set, we compile Arrow using debug flags.
silent (optional bool): If set, does not print so much info.
cores (optional int): Number of cores to compile RADOS-arrow with.
Returns:
`True, admin_node_id` on success, `False, None` otherwise.'''
if not _check_users(reservation):
printe(
'Found different usernames between nodes. All nodes must have the same user login!'
)
return False, None
admin_picked, _ = _pick_admin(reservation, admin=admin_id)
printc('Picked admin node: {}'.format(admin_picked), Color.CAN)
local_connections = connectionwrapper == None
if local_connections:
ssh_kwargs = {
'IdentitiesOnly': 'yes',
'User': admin_picked.extra_info['user'],
'StrictHostKeyChecking': 'no'
}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrapper = get_wrapper(admin_picked,
admin_picked.ip_public,
ssh_params=ssh_kwargs,
silent=silent)
else:
if not connectionwrapper.open:
raise ValueError('Cannot use already closed connection.')
rados_module = _generate_module_rados()
retval = _install_rados(connectionwrapper.connection,
rados_module,
reservation,
install_dir,
arrow_url=arrow_url,
force_reinstall=force_reinstall,
debug=debug,
silent=silent,
cores=cores), admin_picked.node_id
if local_connections:
close_wrappers([connectionwrapper])
return retval
def install_ssh(reservation,
connectionwrappers=None,
key_path=None,
cluster_keypair=None,
silent=False,
use_sudo=defaults.use_sudo()):
'''Installs ssh keys in the cluster for internal traffic.
Warning: Requires that usernames on remote cluster nodes are equivalent.
Args:
reservation (`metareserve.Reservation`): Reservation object with all nodes to install RADOS-Ceph on.
connectionwrappers (optional dict(metareserve.Node, RemotoSSHWrapper)): If set, uses given connections, instead of building new ones.
install_dir (str): Location on remote host to install RADOS-Ceph in.
key_path (optional str): Path to SSH key, which we use to connect to nodes. If `None`, we do not authenticate using an IdentityFile.
cluster_keypair (optional tuple(str,str)): Keypair of (private, public) key to use for internal comms within the cluster. If `None`, a keypair will be generated.
silent (optional bool): If set, does not print so much info.
Returns:
`True` on success, `False` otherwise.'''
if not _check_users(reservation):
printe(
'Found different usernames between nodes. All nodes must have the same user login!'
)
return False
user = list(reservation.nodes)[0].extra_info['user']
local_connections = connectionwrappers == None
if local_connections:
ssh_kwargs = {
'IdentitiesOnly': 'yes',
'User': user,
'StrictHostKeyChecking': 'no'
}
if key_path:
ssh_kwargs['IdentityFile'] = key_path
connectionwrappers = get_wrappers(reservation.nodes,
lambda node: node.ip_public,
ssh_params=ssh_kwargs,
silent=silent)
else:
if not all(x.open for x in connectionwrappers):
raise ValueError(
'SSH installation failed: At least one connection is already closed.'
)
with concurrent.futures.ThreadPoolExecutor(
max_workers=len(reservation)) as executor:
ssh_module = _generate_module_ssh()
futures_ssh_installed = {
node: executor.submit(_installed_ssh,
wrapper.connection,
ssh_module,
keypair=cluster_keypair)
for node, wrapper in connectionwrappers.items()
}
do_install = False
for node, ssh_future in futures_ssh_installed.items():
if not ssh_future.result():
print('SSH keys not installed in node: {}'.format(node))
do_install = True
if do_install:
internal_keypair = cluster_keypair
if not internal_keypair:
internal_keypair = _make_keypair()
futures_ssh_install = {
node: executor.submit(_install_ssh,
wrapper.connection,
ssh_module,
reservation,
internal_keypair,
user,
use_sudo=use_sudo)
for node, wrapper in connectionwrappers.items()
}
state_ok = True
for node, ssh_future in futures_ssh_install.items():
if not ssh_future.result():
printe(
'Could not setup internal ssh key for node: {}'.format(
node))
state_ok = False
if local_connections:
close_wrappers(connectionwrappers)
return state_ok
else:
prints('SSH keys already installed.')
if local_connections:
close_wrappers(connectionwrappers)
return True