def set_new_assignments(self, paths_assignment_info):
logger.info("Set new assignment.")
if self.get_current_reassignment() is not None:
raise Exception("There is already running assignment.")
config_api = ConfigAPI()
consul_api = ConsulAPI()
logger.info("Delete old assignments.")
consul_api.delete_assignments()
session = consul_api.get_new_session_ID(
config_api.get_assignment_session_name(),
configuration().get_node_name(), True)
if consul_api.lock_key(config_api.get_consul_assignment_path(),
session, "root"):
logger.info("Lock assignment root.")
for path_assignment_info in paths_assignment_info:
path_assignment_info.status = ReassignPathStatus.pending
consul_api.set_path_assignment(
path_assignment_info,
self._get_node_session(path_assignment_info.target_node))
logger.info(
"New assignment for {} ,disk {}, from node {} and to node {} with status {}"
.format(path_assignment_info.ip,
path_assignment_info.disk_id,
path_assignment_info.node,
path_assignment_info.target_node,
path_assignment_info.status))
else:
logger.error("Can't lock paths assignment key.")
raise Exception("Can't lock paths assignment key.")
class MangePathAssignment(object):
def __init__(self):
self.__app_conf = ConfigAPI()
self.__context = AssignmentContext()
self.__session_dict = ConsulAPI().get_sessions_dict(
ConfigAPI().get_iscsi_service_session_name())
self.__node_session_dict = dict()
pass
def get_assignments_stats(self):
return self._filter_assignments_stats()
def search_by_disk_name(self, disk_name):
return self._filter_assignments_stats(filter_type=1,
filter_text=disk_name)
def search_by_ip(self, ip):
return self._filter_assignments_stats(filter_type=2, filter_text=ip)
def _filter_assignments_stats(self,
filter_type=0,
filter_text=None,
set_session=False):
__disk_consul_stopped = set()
running_paths = dict()
ceph_api = CephAPI()
consul_api = ConsulAPI()
disk_kvs = consul_api.get_disk_kvs()
# Step 1 get all running paths.
for consul_kv_obj in disk_kvs:
path_key = str(consul_kv_obj.Key).replace(
self.__app_conf.get_consul_disks_path(), "")
disk_id = str(path_key).split('/')[0]
if disk_id in __disk_consul_stopped:
continue
if consul_kv_obj.Value == "disk":
disk_id = str(path_key).split('/')[0]
# Step 2 avoid stopping disks
if str(consul_kv_obj.Flags) == "1":
__disk_consul_stopped.add(disk_id)
continue
running_paths[path_key] = consul_kv_obj
if len(running_paths) == 0:
return AssignmentStats()
# Step 3 get all images metadata
images = ceph_api.get_disks_meta()
assignment_stats = AssignmentStats()
# Step 4 get current reassignments
current_running_assignments = self.get_current_reassignment()
if current_running_assignments is not None:
assignment_stats.is_reassign_busy = True
filter_type = 0 # we will stop any filter and get all data if here is running reassignment
# Step 5 fill paths assignment info
for path_key, consul_kv_obj in running_paths.iteritems():
disk_id = str(path_key).split('/')[0]
disk = next((img for img in images if img.id == disk_id), None)
if disk is None:
continue
disk_path = Path()
path_index = int(str(path_key).split(disk_id + "/")[1])
path_str = disk.paths[path_index - 1]
disk_path.load_json(json.dumps(path_str))
path_assignment_info = PathAssignmentInfo()
path_assignment_info.interface = disk_path.eth
if disk_path.vlan_id:
path_assignment_info.interface = disk_path.eth + "." + disk_path.vlan_id
path_assignment_info.ip = disk_path.ip
path_assignment_info.disk_name = disk.disk_name
path_assignment_info.disk_id = disk_id
path_assignment_info.index = path_index
current_path = None
if current_running_assignments is not None:
current_path = current_running_assignments.get(disk_path.ip)
if hasattr(consul_kv_obj,
"Session") and self.__session_dict.has_key(
consul_kv_obj.Session):
# Fill status and node name for started paths
path_assignment_info.node = self.__session_dict.get(
consul_kv_obj.Session).Node
if current_running_assignments is not None:
if current_path is not None and current_path.status != -1:
path_assignment_info.status = current_path.status
path_assignment_info.target_node = current_path.target_node
if set_session:
# session refers to the node that lock this path assignment,This property helps to know the
# status of path and the node will handle this path
path_assignment_info.session = current_path.session
elif current_path:
path_assignment_info.node = current_path.node
path_assignment_info.target_node = current_path.target_node
path_assignment_info.status = current_path.status
if set_session:
path_assignment_info.session = current_path.session
# Step 6 search or get all
if filter_type == 1 and filter_text is not None and len(
str(filter_text).strip()) > 0: # by disk name
if filter_text.strip().lower(
) in path_assignment_info.disk_name.lower():
assignment_stats.paths.append(path_assignment_info)
elif filter_type == 2 and filter_text is not None and len(
str(filter_text).strip()) > 0: # by ip
if filter_text.strip() == path_assignment_info.ip.strip():
assignment_stats.paths.append(path_assignment_info)
break
else:
assignment_stats.paths.append(path_assignment_info)
# Step 7 set all online nodes
assignment_stats.nodes = self._get_nodes()
return assignment_stats
def get_current_reassignment(self):
paths = ConsulAPI().get_assignments()
if paths is not None:
for ip, path_assignment_info in paths.iteritems():
if not hasattr(path_assignment_info, "session"):
logger.info("Path {} not locked by node.".format(
path_assignment_info.ip))
if not hasattr(
path_assignment_info,
"session") and path_assignment_info.status not in [
ReassignPathStatus.succeeded,
ReassignPathStatus.failed
]:
path_assignment_info.status = ReassignPathStatus.failed
return paths
def set_new_assignments(self, paths_assignment_info):
logger.info("Set new assignment.")
if self.get_current_reassignment() is not None:
raise Exception("There is already running assignment.")
config_api = ConfigAPI()
consul_api = ConsulAPI()
logger.info("Delete old assignments.")
consul_api.delete_assignments()
session = consul_api.get_new_session_ID(
config_api.get_assignment_session_name(),
configuration().get_node_name(), True)
if consul_api.lock_key(config_api.get_consul_assignment_path(),
session, "root"):
logger.info("Lock assignment root.")
for path_assignment_info in paths_assignment_info:
path_assignment_info.status = ReassignPathStatus.pending
consul_api.set_path_assignment(
path_assignment_info,
self._get_node_session(path_assignment_info.target_node))
logger.info(
"New assignment for {} ,disk {}, from node {} and to node {} with status {}"
.format(path_assignment_info.ip,
path_assignment_info.disk_id,
path_assignment_info.node,
path_assignment_info.target_node,
path_assignment_info.status))
else:
logger.error("Can't lock paths assignment key.")
raise Exception("Can't lock paths assignment key.")
def run(self):
cmd = "python {} server &".format(
ConfigAPI().get_assignment_script_path())
call_cmd(cmd)
def _get_nodes(self):
consul_api = ConsulAPI()
# Get all PetaSAN nodes[management or storage].
node_list = consul_api.get_node_list()
# Get online nodes from consul.
consul_members = consul_api.get_consul_members()
petasan_node_list = []
for i in node_list:
if not i.is_iscsi:
continue
if i.name in consul_members:
petasan_node_list.append(i.name)
return petasan_node_list
def remove_assignment(self):
consul_api = ConsulAPI()
if consul_api.get_assignments() is not None:
consul_api.delete_assignments()
def auto(self, type=1):
logger.info("User start auto reassignment paths.")
assignments_stats = self.get_assignments_stats()
if assignments_stats.is_reassign_busy:
logger.error("There is already reassignment running.")
raise Exception("There is already reassignment running.")
ConsulAPI().drop_all_node_sessions(
self.__app_conf.get_consul_assignment_path(),
configuration().get_node_name())
sleep(3)
assignments_stats.paths = [
path for path in assignments_stats.paths
if len(path.node.strip()) > 0 and path.status == -1
]
self.__context.paths = assignments_stats.paths
self.__context.nodes = assignments_stats.nodes
for plugin in self._get_new_plugins_instances(auto_plugins):
if plugin.is_enable() and plugin.get_plugin_id() == type:
paths_assignments = plugin.get_new_assignments()
if len(paths_assignments) == 0:
logger.info("There is no node under average.")
return
self.set_new_assignments(paths_assignments)
break
self.run()
def manual(self, paths_assignment_info, assign_to="auto"):
assignments_stats = self.get_assignments_stats()
if assignments_stats.is_reassign_busy:
logger.error("There is already reassignment running.")
raise Exception("There is already reassignment running.")
ConsulAPI().drop_all_node_sessions(
self.__app_conf.get_consul_assignment_path(),
configuration().get_node_name())
sleep(3) # Wait to be sure the session dropped
if assign_to == "auto":
logger.info(
"User start auto reassignment paths for selected paths.")
assignments_stats.paths = [
path for path in assignments_stats.paths
if len(path.node.strip()) > 0 and path.status == -1
]
self.__context.paths = assignments_stats.paths
self.__context.nodes = assignments_stats.nodes
self.__context.user_input_paths = paths_assignment_info
for plugin in self._get_new_plugins_instances(auto_plugins):
if plugin.is_enable() and plugin.get_plugin_id() == 1:
paths_assignments = plugin.get_new_assignments()
self.set_new_assignments(paths_assignments)
logger.info(
"User start auto reassignment paths for selected paths."
)
self.run()
break
pass
else:
for path_assignment_info in paths_assignment_info:
path_assignment_info.target_node = assign_to
path_assignment_info.status = ReassignPathStatus.pending
logger.info(
"User start manual reassignment paths for selected paths.")
self.set_new_assignments(paths_assignment_info)
self.run()
def process(self):
logger.info("Start process reassignments paths.")
max_retry = 100
current_reassignments = self.get_current_reassignment()
config = configuration()
assignment_script_path = ConfigAPI().get_assignment_script_path()
if current_reassignments is None:
return
for ip, path_assignment_info in current_reassignments.iteritems():
logger.info("process path {} and its status is {}".format(
ip, path_assignment_info.status))
if path_assignment_info.status == ReassignPathStatus.pending:
logger.info(
"Move action,try clean disk {} path {} remotely on node {}."
.format(path_assignment_info.disk_name,
path_assignment_info.disk_id,
path_assignment_info.node))
status = False
try:
cmd = "python {} path_host -ip {} -disk_id {}".format(
assignment_script_path, path_assignment_info.ip,
path_assignment_info.disk_id)
out, err = ssh().exec_command(path_assignment_info.node,
cmd)
logger.info(cmd)
# self.clean_source_node(path_assignment_info.ip,path_assignment_info.disk_id)
except Exception as ex:
logger.exception(ex.message)
out = ""
if str(out).strip() == "0":
logger.info("Move action passed")
status = True
current_path_assignment_info = None
if status:
for i in xrange(0, max_retry):
logger.debug(
"Wait to update status of path {}.".format(
path_assignment_info.ip))
sleep(0.25)
reassignments = self.get_current_reassignment()
if reassignments:
current_path_assignment_info = reassignments.get(
path_assignment_info.ip)
if current_path_assignment_info and current_path_assignment_info.status == ReassignPathStatus.moving:
continue
else:
logger.info(
"Process completed for path {} with status {}."
.format(
current_path_assignment_info.ip,
current_path_assignment_info.status))
break
if current_path_assignment_info and current_path_assignment_info.status == ReassignPathStatus.moving:
self.update_path(current_path_assignment_info,
ReassignPathStatus.failed)
logger.info(
"Move action,failed ,disk {} path {}.".format(
path_assignment_info.disk_name,
path_assignment_info.disk_id,
path_assignment_info.node))
else:
self.update_path(path_assignment_info,
ReassignPathStatus.failed)
logger.info(
"Move action ,failed to clean disk {} path {} remotely on node ."
.format(path_assignment_info.disk_name,
path_assignment_info.disk_id,
path_assignment_info.node))
sleep(10) # wait for display status to user if needed
logger.info("Process completed.")
self.remove_assignment()
ConsulAPI().drop_all_node_sessions(
self.__app_conf.get_consul_assignment_path(),
config.get_node_name())
def _clean_iscsi_config(self, disk_id, path_index, iqn):
logger.debug("Move action ,start clean disk {} path {}.".format(
disk_id, path_index))
lio_api = LioAPI()
try:
# Get tpgs for iqn.
tpgs = lio_api.get_iqns_with_enabled_tpgs().get(iqn, None)
if not iqn or not tpgs or len(tpgs) == 0:
logger.info("Move action ,could not find ips for %s " %
disk_id)
# Remove the assigned ips from our interfaces
elif tpgs and len(tpgs) > 0:
# Get assigned ips for each path.
for tpg, ips in tpgs.iteritems():
if tpg == str(path_index + 1):
lio_api.disable_path(iqn, tpg)
logger.info(
"Move action,cleaned disk {} path {}.".format(
disk_id, path_index))
break
except Exception as e:
logger.error("Move action,could not clean disk path for %s" %
disk_id)
return False
logger.debug("Move action end clean disk {} path {}.".format(
disk_id, path_index))
return True
def clean_source_node(self, ip, disk_id):
if not self.update_path(ip, ReassignPathStatus.moving):
return False
# pool = CephAPI().get_pool_bydisk(disk_id)
pool = self._get_pool_by_disk(disk_id)
if not pool:
logger.error('Could not find pool for disk ' + disk_id)
return False
disk = CephAPI().get_disk_meta(disk_id, pool)
paths_list = disk.paths
disk_path = None
path_index = -1
for i in xrange(0, len(paths_list)):
path_str = paths_list[i]
path = Path()
path.load_json(json.dumps(path_str))
if path.ip == ip:
disk_path = path
path_index = i
break
if disk_path:
self._clean_iscsi_config(disk_id, path_index, disk.iqn)
network = Network()
NetworkAPI().delete_ip(path.ip, path.eth, path.subnet_mask)
if network.is_ip_configured(ip):
logger.error(
"Move action,cannot clean newtwork config for disk {} path {}."
.format(disk_id, path_index))
self.update_path(ip, ReassignPathStatus.failed)
return False
logger.info(
"Move action,clean newtwork config for disk {} path {}.".
format(disk_id, path_index))
key = self.__app_conf.get_consul_disks_path(
) + disk_id + "/" + str(path_index + 1)
consul_api = ConsulAPI()
session = self._get_node_session(configuration().get_node_name())
if ConsulAPI().is_path_locked_by_session(key, session):
consul_api.release_disk_path(key, session, None)
logger.info("Move action,release disk {} path {}.".format(
disk_id, path_index + 1))
else:
self.update_path(ip, ReassignPathStatus.failed)
return False
return True
def update_path(self, ip, status):
logger.info("Updating path {} status to {} ".format(ip, status))
current_reassignments = self.get_current_reassignment()
if current_reassignments:
path_assignment_info = current_reassignments.get(ip)
if path_assignment_info:
path_assignment_info.status = status
if ConsulAPI().update_path_assignment(path_assignment_info):
logger.info("Path {} status updated to {} ".format(
ip, status))
return True
logger.info("Path {} status failed to update status to {} ".format(
ip, status))
return False
def _get_new_plugins_instances(self, modules):
plugins = []
for cls in modules:
try:
# import plugins module
mod_obj = __import__(cls)
for i in str(cls).split(".")[1:]:
mod_obj = getattr(mod_obj, i)
# Find all plugins in module and create instances
for mod_prop in dir(mod_obj):
# Ignore private
if not str(mod_prop).startswith("__"):
attr = getattr(mod_obj, mod_prop)
attr_str = str(attr)
attr_type_str = str(type(attr))
# Find plugin from type ABCMeta , plugin class name contains 'plugin' and not contains base
if attr_type_str.find(
"ABCMeta") > -1 and attr_str.find(
"Base") == -1 and attr_str.find("Plugin"):
instance = attr(self.__context)
plugins.append(instance)
except Exception as e:
logger.error("Error load plugin {}.".format(cls))
return plugins
def get_forced_paths(self):
paths = None
assignments = self._filter_assignments_stats(set_session=True)
if not assignments.is_reassign_busy:
return paths
for path_assignment_info in assignments.paths:
if path_assignment_info.status == ReassignPathStatus.moving and hasattr(
path_assignment_info, "session"):
if paths is None:
paths = dict()
paths[path_assignment_info.disk_id + "/" +
str(path_assignment_info.index)] = path_assignment_info
return paths
def _get_node_session(self, node_name):
logger.info(self.__node_session_dict)
if self.__session_dict:
session = self.__node_session_dict.get(node_name)
if session is not None:
return session
else:
for sess, node in self.__session_dict.iteritems():
if node.Node == node_name:
self.__node_session_dict[node] = sess
return sess
def _get_pool_by_disk(self, disk_id):
consul_api = ConsulAPI()
ceph_api = CephAPI()
pool = consul_api.get_disk_pool(disk_id)
if pool:
logger.info('Found pool:{} for disk:{} via consul'.format(
pool, disk_id))
return pool
pool = ceph_api.get_pool_bydisk(disk_id)
if pool:
logger.info('Found pool:{} for disk:{} via ceph'.format(
pool, disk_id))
return pool
logger.error('Could not find pool for disk ' + disk_id)
return None
