def _generate_ceph_conf(self):
"""Creates from scratch a ceph.conf that matches the current state of
the depot."""
self.config = TCCephConf()
self.config.create_default(self.uuid)
daemon_list = self.get_daemon_list()
for daemon in daemon_list:
daemon.add_to_config(self.config)
class Depot(object):
"""Controls and maintains the state of a depot
Dev note: Logic, statements, operations and commands that affect the entire
cluster belong in this class. If it affects only a single daemon, go to
the Daemon class. If it affects more than one depot, go to the
TcdsService class.
"""
uuid = None
_daemon_map = None
service = None
utils = None
var = None
CONSTANTS = {
'STATE_OFFLINE': 0,
'STATE_ONLINE': 1
}
config_file_path = None
config = None
class DepotInfo(dict):
"""Class for retrieving information about a depot
Usage: call populate() after creating an instance, passing it a depot
Keys: Type: Values:
'depot_id' : string - uuid of the depot
'depot_replication' : int - replication factor of the depot
'depot_state' : string - state of the depot
'depot_capacity' : string - capacity of the depot in bytes
'depot_usage' : string - used space of the depot in bytes
"""
def __init__(self):
dict.__init__(self)
@staticmethod
def get_state_string(state):
"""Maps internal depot state representation to strings
@type state depot internal state reprentation
(int at the time of writing)
@param state depot state
@rtype string
@return human readable string of the depot state
"""
if state == Depot.CONSTANTS['STATE_OFFLINE']:
return 'not ready'
elif state == Depot.CONSTANTS['STATE_ONLINE']:
return 'ready'
def populate(self, depot):
"""Fills in predefined fields with information from the given
depot"""
self['depot_id'] = depot.uuid
self['depot_replication'] = depot.var.get_depot_replication_factor(depot)
self['depot_state'] = self.get_state_string(depot.var.get_depot_state(depot))
if self['depot_state'] == 'ready':
libceph = depot.utils.get_libceph(depot.config_file_path)
if libceph is not None:
(avail, total) = libceph.df()
self['depot_capacity'] = str(total)
self['depot_usage'] = str(total - avail)
return
else:
depot.utils.dout(logging.INFO, 'returning fake info because get_libceph failed')
else:
depot.utils.dout(logging.INFO, 'returning fake info because depot is not ready')
self['depot_capacity'] = 0
self['depot_usage'] = 0
def __init__(self, service, uuid):
self.service = service
self.uuid = uuid
self._daemon_map = {}
self.utils = service.utils
self.var = service.var
self.config_file_path = os.path.join(self.utils.CONFIG_FILE_PATH_PREFIX, '%s.conf' % self.uuid)
def _load_saved_state(self):
"""Read and reproduce the state of the cluster from this instance's
varstore."""
daemon_spec_list = self.var.get_depot_daemon_list(self)
for daemon_spec in daemon_spec_list:
daemon = self._new_daemon_instance(daemon_spec['type'], daemon_spec['uuid'])
self._daemon_map[daemon.uuid] = daemon
daemon._load_saved_state()
self._generate_ceph_conf()
def setup(self):
"""Prepares the depot for operation.
Call after __init__
"""
self._generate_ceph_conf()
def _generate_ceph_conf(self):
"""Creates from scratch a ceph.conf that matches the current state of
the depot."""
self.config = TCCephConf()
self.config.create_default(self.uuid)
daemon_list = self.get_daemon_list()
for daemon in daemon_list:
daemon.add_to_config(self.config)
def get_info(self):
"""Return a DepotInfo instance with this depot's information"""
depot_info = self.DepotInfo()
depot_info.populate(self)
return depot_info
def _new_daemon_instance(self, daemon_type, uuid):
"""Create and return a new daemon instance
@type daemon_type string
@param daemon_type the type of daemon to create. Valid types at the
time of writing include 'mon','mds','osd'
@type uuid string
@param uuid uuid of the new daemon
@rtype instance of a Daemon subclass
@return the newly created Daemon
"""
if daemon_type == 'mon':
return Mon(self, uuid)
elif daemon_type == 'mds':
return Mds(self, uuid)
elif daemon_type == 'osd':
return Osd(self, uuid)
else:
raise ValueError('storage_roles should be one of mon, mds, osd')
def add_daemons(self, daemon_spec_list):
"""Create daemons and add daemons to the depot
Creates, registers, sets up, and if conditions allow, starts the given
daemons under the depot.
@type daemon_spec_list list
@param daemon_spec_list list of daemon specification dictionaries. Each
daemon specification should contain the fields 'type',
'host' and 'uuid'; where 'type' is a valid daemon type,
'host is the uuid of the daemon's host machine, and 'uuid'
is the uuid string of the new daemon
@rtype list
@return a list of the daemons instances created
"""
orig_num_osd = self._get_daemon_count()['num_osd']
new_daemon_list = []
new_mons = []
for daemon_spec in daemon_spec_list:
# Create
new_daemon = self._new_daemon_instance(daemon_spec['type'],
daemon_spec['uuid'])
new_daemon_list.append(new_daemon)
if new_daemon.TYPE == 'mon':
new_mons.append(new_daemon)
new_ceph_name = self._get_next_ceph_name_for(new_daemon.TYPE)
# Register
self.var.add_daemon(new_daemon, daemon_spec['uuid'],
daemon_spec['host'], new_ceph_name)
self._daemon_map[daemon_spec['uuid']] = new_daemon
if self.get_state() == self.CONSTANTS['STATE_OFFLINE']:
daemon_count = self._get_daemon_count()
if Depot._get_meets_min_requirements(replication=self.var.get_depot_replication_factor(self), **daemon_count):
self.activate()
else:
self.utils.dout(logging.DEBUG, 'Add complete, not activating (daemon count=%s, replication=%s)' % (daemon_count, self.var.get_depot_replication_factor(self)))
elif self.get_state() == self.CONSTANTS['STATE_ONLINE']:
if self._check_depot() is not True:
raise TcdsError('aborting add because depot does not appear to be usable')
# Setup
old_config = copy.deepcopy(self.config)
if len(new_mons) > 0:
for daemon in new_mons:
cmd = 'ceph -c %s mon add %s %s:6789' % (
self.config_file_path, daemon.get_ceph_name(),
daemon.get_host_ip())
self.utils.run_shell_command(cmd)
self._wait_for_map_update()
"""NOTE: Something bad might happen if the number of
monitors being added exceeds the number of original
monitors. E.g. if there are three monitors and four more
are added, when adding the fourth the monmap proposal should
fail because only three out of six monitors are active."""
for daemon in new_daemon_list:
daemon.setup(old_config)
for daemon in new_daemon_list:
daemon.add_to_config(self.config)
for daemon in new_daemon_list:
daemon.set_config(self.config)
num_osd = self._get_daemon_count()['num_osd']
if num_osd > orig_num_osd:
cmd = 'ceph -c %s osd setmaxosd %s' % (self.config_file_path, num_osd)
self.utils.run_shell_command(cmd)
new_crushmap = self._generate_crushmap()
cmd = 'ceph -c %s osd setcrushmap -i %s' % (self.config_file_path, new_crushmap)
self.utils.run_shell_command(cmd)
# Start
for daemon in new_daemon_list:
daemon.activate()
if self._check_depot() is not True:
self.utils.dout(logging.WARNING, 'check depot failed during add, hopefully this does not matter...')
self.write_config()
return new_daemon_list
def _wait_for_map_update(self):
"""Returns only after the monitors have updated their maps"""
exec 'import time'
time.sleep(5) # FIXME: ideas welcome...
def _generate_crushmap(self):
"""Returns the path to a crushmap that matches the current state
@rtype string
@return path to an encoded crushmap ready for injection
"""
num_osd = self._get_daemon_count()['num_osd']
cmd = 'osdmaptool --createsimple %s --clobber /tmp/osdmap.junk --export-crush /tmp/crush.new' % (num_osd,)
self.utils.run_shell_command(cmd)
return '/tmp/crush.new'
def remove_nodes(self, node_list, force=False):
"""Removes all daemons whose host is one of the hosts given
By default, this operation will check that after performing the
removal, the depot will remain healty (as per the definition in
Depot._get_meets_min_requirements()). If the check fails, the
opertaion will be aborted.
@type node_list list
@param node_list list of host machine uuid strings on which to remove
daemons
@type force boolean
@param force if True, does not perform checks and will always
attempt to remove the target daemons
@rtype list
@return a list containing the daemon instances removed
"""
daemon_list = self.get_daemon_list()
daemon_count = self._get_daemon_count()
remove_pending = []
processed_nodes = []
for node_id in node_list:
if node_id in processed_nodes: # skip duplicates
continue
processed_nodes.append(node_id)
for daemon in daemon_list:
if daemon.get_host_id() == node_id:
remove_pending.append(daemon)
daemon_count['num_'+daemon.TYPE] = daemon_count['num_'+daemon.TYPE] - 1
if force or self._get_meets_min_requirements(replication=self.var.get_depot_replication_factor(self), **daemon_count):
self._del_daemons(remove_pending)
return remove_pending
else:
raise TcdsError('remove_nodes: remove aborted because depot daemon count will fall below min requirements')
def _del_daemons(self, daemons_to_remove):
"""Deletes and unregisters the given daemons
@type daemons_to_remove list
@param daemons_to_remove list of daemon instances to remove
"""
for daemon in daemons_to_remove:
daemon.delete()
daemon.del_from_config(self.config)
del self._daemon_map[daemon.uuid]
self.var.remove_daemons(daemons_to_remove)
self.write_config()
def get_state(self):
"""Return the value of the state of the depot"""
return self.var.get_depot_state(self)
def set_state(self, state):
"""Set the value of the state of the depot"""
self.var.set_depot_state(self, state)
def get_daemon_list(self, daemon_type='all'):
"""Return a list of all daemons whose type is daemon_type
@type daemon_type string
@param daemon_type the type of the daemons to return or 'all'
@rtype list
@return list of all daemons in the depot matching daemon_type
"""
if daemon_type == 'all':
return self._daemon_map.values()
return_list = []
for daemon in self._daemon_map.values():
if daemon.TYPE == daemon_type:
return_list.append(daemon)
return return_list
def _get_daemon_count(self):
"""Count the number of daemons (by type)
@rtype dict
@return dict with keys num_<type> and values being the count
"""
daemon_list = self.get_daemon_list()
daemon_count = {'num_mon': 0, 'num_mds':0, 'num_osd':0}
for daemon in daemon_list:
daemon_count['num_'+daemon.TYPE] = daemon_count['num_'+daemon.TYPE] + 1
return daemon_count
def apply_replication_factor(self, factor):
"""Apply the replication factor setting to the depot
@type factor int
@param factor the replication factor to apply
"""
assert self.config_file_path is not None, 'config_file_path not set'
cmd = "ceph -c %s osd pool set metadata size %s" % (self.config_file_path, factor)
self.utils.run_shell_command(cmd)
cmd = "ceph -c %s osd pool set data size %s" % (self.config_file_path, factor)
self.utils.run_shell_command(cmd)
@staticmethod
def _get_meets_min_requirements(replication, num_mon, num_mds, num_osd):
"""Return whether the given number of daemons can form a healthy
cluster
@type replication int
@param replication replication factor of the depot to form
@type num_mon int
@param num_mon number of monitors in the depot to form
@type num_mds int
@param num_mds number of mds in the depot to form
@type num_osd int
@param num_osd number of osd in the depot to form
@rtype boolean
@return True if a healthy depot can be formed
"""
if type(replication) is not int \
or type(num_mon) is not int \
or type(num_mds) is not int \
or type(num_osd) is not int:
raise TypeError('int arguments expected, got %s, %s, %s, %s' %
(type(replication), type(num_mon), type(num_mds), type(num_osd)))
if replication < 1 or num_mon < 0 or num_mds < 0 or num_osd < 0:
raise ValueError('got (rep,mon,mds,osd)=(%s,%s,%s,%s)' % (replication, num_mon, num_mds, num_osd))
if num_mon >= 3 and num_mds >= 2 and num_osd >= replication:
return True
else:
return False
def _get_next_ceph_name_for(self, daemon_type):
"""Return the next ceph name that should be assigned to a daemon of the
given type
@type daemon_type string
@param daemon_type the daemon type for which to fetch the next name
@rtype string
@return the name that should be assigned to the daemon
"""
daemon_list = self.get_daemon_list(daemon_type)
in_use_names = []
for daemon in daemon_list:
daemon_name = daemon.get_ceph_name()
if daemon_name.isdigit():
in_use_names.append(daemon_name)
in_use_names.sort()
for i in range(len(in_use_names)):
if in_use_names[i].isdigit() and int(in_use_names[i]) != i:
return i
return len(in_use_names)
def _check_ceph_ids_are_consecutive(self):
"""Return whether the ceph names currently in use are consecutive
Consecutivity is checked individually for each daemon type and the
function will only evaluate to True if all daemon types are
consecutive. However, names that cannot be interpreted as int (e.g. 'a'
) are ignored and do not affect consecutivity.
@rtype boolean
@return True if consecutive
"""
daemon_list = self.get_daemon_list()
daemon_list.sort(Mon.cmp_name)
next_id = {'mon': 0, 'mds': 0, 'osd': 0}
for daemon in daemon_list:
daemon_name = daemon.get_ceph_name()
if not daemon_name.isdigit():
continue
if next_id[daemon.TYPE] == int(daemon_name):
next_id[daemon.TYPE] = next_id[daemon.TYPE] + 1
else:
return False
return True
def write_config(self):
"""Write the current config to disk"""
assert self.config_file_path is not None, 'config_file_path not set'
with open(self.config_file_path, 'wb') as config_file:
self.config.write(config_file)
def update_daemon_configs(self, daemon_list=None):
"""Update all daemons (or the given daemons) with the current config
and on each daemon's host, write the config to disk."""
if daemon_list == None:
daemon_list = self.get_daemon_list()
for daemon in daemon_list:
daemon.set_config(self.config)
daemon.write_config()
def activate(self, first=True):
"""Activate (start) the depot (for the first time)
This function will activate the depot by calling activate() of each
daemon and taking necessary pre-activate and post-activate actions.
If the depot is being activated for the first time, pass first=True
(which is the default). This function will abort if the depot state
is not equal to STATE_OFFLINE.
NB: If this function is called with first=True, the depot will be
REFORMED and all data will be LOST.
@type first boolean
@param first pass True if this is the first time the depot is being
activated. Or if you want to reformat...
@rtype boolean
@return True if the operation completed successfully
"""
if self.get_state() != self.CONSTANTS['STATE_OFFLINE']:
return False
if first:
daemon_list = self.get_daemon_list()
next_id = {'mon': 0, 'mds': 0, 'osd': 0}
for daemon in daemon_list:
daemon.set_ceph_name(next_id[daemon.TYPE])
daemon.add_to_config(self.config)
next_id[daemon.TYPE] = next_id[daemon.TYPE] + 1
self.write_config()
for daemon in daemon_list:
daemon.set_config(self.config)
if first:
cmd = "mkcephfs -c %s --allhosts" % (self.config_file_path, )
self.utils.run_shell_command(cmd)
for daemon in daemon_list:
daemon.activate()
if first:
self.apply_replication_factor(self.var.get_depot_replication_factor(self))
self.set_state(self.CONSTANTS['STATE_ONLINE'])
return True
def deactivate(self):
"""Deactivate (stop) the depot and all its daemons"""
for daemon in self.get_daemon_list():
daemon.deactivate()
self.set_state(self.CONSTANTS['STATE_OFFLINE'])
def delete(self):
"""Delete the depot and all its daemons"""
self._del_daemons(self.get_daemon_list())
os.remove(self.config_file_path)
def _check_depot(self):
"""Returns whether the depot is service
The check is performed by attempting to write a file to the filesystem
@rtype boolean
@return True if the depot is in service
"""
libceph = self.utils.get_libceph(self.config_file_path)
retries = 3
if libceph is None:
return True
testfile_name = "tcds.api.testfile"
for _ in range(retries):
fd = libceph.open(testfile_name, os.O_CREAT|os.O_RDWR, stat.S_IRWXU)
if fd >= 0:
break
else:
raise TcdsError('unable to create testfile')
if fd >= 0:
if libceph.write(fd, testfile_name, 0) < 0:
raise TcdsError('write failed')
libceph.fsync(fd, False)
libceph.close(fd)
if libceph.unlink(testfile_name) != 0:
raise TcdsError('unlink failed')
return True
