def learn_fs_targets(self):
for host, host_data in self.all_hosts_data.items():
for local_info in host_data["local_targets"]:
if not local_info["mounted"]:
log.warning("Ignoring unmounted target %s on host %s" %
(local_info["name"], host))
continue
name = local_info["name"]
device_node_paths = local_info["device_paths"]
uuid = local_info["uuid"]
if name.find("-MDT") != -1:
klass = ManagedMdt
elif name.find("-OST") != -1:
klass = ManagedOst
elif name == "MGS":
continue
else:
raise NotImplementedError()
try:
mgs = self._target_find_mgs(host, local_info)
except ManagedMgs.DoesNotExist:
log.warning("Can't find MGS for target %s on %s" %
(name, host))
continue
fsname, index_str = re.search("([\w\-]+)-(\w)+", name).groups()
index = int(index_str, 16)
# Create Filesystem objects if we've not seen this FS before.
(filesystem,
created) = ManagedFilesystem.objects.get_or_create(
name=fsname, mgs=mgs)
self.discovered_filesystems.add(filesystem)
if created:
self.created_filesystems.append(filesystem)
filesystem.immutable_state = True
filesystem.save()
log.info("Found filesystem '%s'" % fsname)
ObjectCache.add(ManagedFilesystem, filesystem)
try:
klass.objects.get(uuid=uuid)
except ManagedTarget.DoesNotExist:
# Fall through, no targets with that name exist on this MGS
volumenode = self._get_volume_node(host, device_node_paths)
target = klass(
uuid=uuid,
name=name,
filesystem=filesystem,
state="mounted",
volume=volumenode.volume,
index=index,
immutable_state=True,
)
target.save()
log.debug("%s" %
[mt.name for mt in ManagedTarget.objects.all()])
log.info("%s %s %s" % (mgs.id, name, device_node_paths))
log.info("Found %s %s" % (klass.__name__, name))
self.created_targets.append(target)
ObjectCache.add(ManagedTarget, target.managedtarget_ptr)
def run(self, all_hosts_data):
""":param all_hosts_data: Dict of ManagedHost to detect-scan output"""
# Must be run in a transaction to avoid leaving invalid things
# in the DB on failure.
assert transaction.is_managed()
self.all_hosts_data = all_hosts_data
# Create ManagedMgs objects
log.debug(">>learn_mgs_targets")
self.learn_mgs_targets()
# Create ManagedTargetMount objects
log.debug(">>learn_mgs_target_mounts")
self.learn_target_mounts()
# Create ManagedMdt and ManagedOst objects
log.debug(">>learn_fs_targets")
self.learn_fs_targets()
# Create ManagedTargetMount objects
log.debug(">>learn_target_mounts")
self.learn_target_mounts()
# Assign a valid primary mount point,
# and remove any targets which don't have a primary mount point
for target in self.created_mgss + self.created_targets:
if self.learn_primary_target(target):
for tm in target.managedtargetmount_set.all():
self._learn_event(tm.host, target)
else:
self.log(help_text["found_no_primary_mount_point_for_target"] %
(target.target_type(), target))
target.mark_deleted()
if not self.created_filesystems:
self.log(help_text["discovered_no_new_filesystem"])
else:
# Remove any Filesystems with zero MDTs or zero OSTs, or set state
# of a valid filesystem
for fs in self.created_filesystems:
mdt_count = ManagedMdt.objects.filter(filesystem=fs).count()
ost_count = ManagedOst.objects.filter(filesystem=fs).count()
if not mdt_count:
self.log(help_text["found_not_TYPE_for_filesystem"] %
("MDT", fs.name))
fs.mark_deleted()
elif not ost_count:
self.log(help_text["found_not_TYPE_for_filesystem"] %
("OST", fs.name))
fs.mark_deleted()
else:
self.log(help_text[
"discovered_filesystem_with_n_MDTs_and_n_OSTs"] %
(fs.name, mdt_count, ost_count))
if set([t.state
for t in fs.get_targets()]) == set(['mounted']):
fs.state = 'available'
fs.save()
first_target = fs.get_filesystem_targets()[0]
self._learn_event(first_target.primary_host, first_target)
if not self.created_mgss:
self.log(help_text['discovered_no_new_target'] %
ManagedMgs().target_type().upper())
else:
for mgt in self.created_mgss:
self.log(
help_text['discovered_target'] %
(mgt.target_type().upper(), mgt.name, mgt.primary_host))
ObjectCache.add(ManagedTarget, mgt.managedtarget_ptr)
# Bit of additional complication so we can print really cracking messages, and detailed messages.
for target in [ManagedMdt(), ManagedOst()]:
if target.target_type() not in [
target.target_type() for target in self.created_targets
]:
self.log(help_text['discovered_no_new_target'] %
target.target_type().upper())
for target in self.created_targets:
self.log(help_text['discovered_target'] %
(target.target_type().upper(), target.name,
target.primary_host))
def create_simple_filesystem(self, host):
from chroma_core.models import ManagedMgs, ManagedMdt, ManagedOst, ManagedFilesystem, ManagedTargetMount
self.mgt, _ = ManagedMgs.create_for_volume(
synthetic_volume_full(host).id, name="MGS")
self.fs = ManagedFilesystem.objects.create(mgs=self.mgt, name="testfs")
ObjectCache.add(ManagedFilesystem, self.fs)
ObjectCache.add(ManagedTarget,
ManagedTarget.objects.get(id=self.mgt.id))
self.mdt, _ = ManagedMdt.create_for_volume(
synthetic_volume_full(host).id, filesystem=self.fs)
self.ost, _ = ManagedOst.create_for_volume(
synthetic_volume_full(host).id, filesystem=self.fs)
ObjectCache.add(ManagedTarget,
ManagedTarget.objects.get(id=self.mdt.id))
ObjectCache.add(ManagedTarget,
ManagedTarget.objects.get(id=self.ost.id))
ObjectCache.add(ManagedTargetMount,
ManagedTargetMount.objects.get(target_id=self.mgt.id))
ObjectCache.add(ManagedTargetMount,
ManagedTargetMount.objects.get(target_id=self.mdt.id))
ObjectCache.add(ManagedTargetMount,
ManagedTargetMount.objects.get(target_id=self.ost.id))
def _create_simple_synthetic_object(class_, **kwargs):
synthetic_object = class_(**kwargs)
synthetic_object.save()
ObjectCache.add(class_, synthetic_object)
return synthetic_object
def create_simple_fs(fs_name="testfs"):
# Create the MGT
mgt = ManagedMgs.objects.create(
id=1,
state_modified_at="2020-11-11T23:52:23.938603+00:00",
state="unmounted",
immutable_state=False,
name="MGS",
uuid="uuid_mgt",
ha_label=None,
inode_size=None,
bytes_per_inode=None,
inode_count=None,
reformat=False,
not_deleted=True,
)
mgt.save()
ObjectCache.add(ManagedTarget, mgt.managedtarget_ptr)
# Create a filesystem using the mgt
fs = ManagedFilesystem.objects.create(mgs=mgt, name=fs_name, id=1, mdt_next_index=1, ost_next_index=1)
ObjectCache.add(ManagedFilesystem, fs)
# Create the mdt
mdt = ManagedMdt.objects.create(
id=2,
index=0,
filesystem_id=1,
state_modified_at="2020-11-11T23:52:23.938603+00:00",
state="unmounted",
immutable_state=False,
name="{}-MDT0000".format(fs_name),
uuid="uuid_mdt",
ha_label=None,
inode_size=None,
bytes_per_inode=None,
inode_count=None,
reformat=False,
not_deleted=True,
)
mdt.save()
ObjectCache.add(ManagedTarget, mdt.managedtarget_ptr)
# Create the ost
ost = ManagedOst.objects.create(
id=3,
index=0,
filesystem_id=1,
state_modified_at="2020-11-11T23:52:23.938603+00:00",
state="unmounted",
immutable_state=False,
name="{}-OST0000".format(fs_name),
uuid="uuid_ost0",
ha_label=None,
inode_size=None,
bytes_per_inode=None,
inode_count=None,
reformat=False,
not_deleted=True,
)
ost.save()
ObjectCache.add(ManagedTarget, ost.managedtarget_ptr)
return (mgt, fs, mdt, ost)
def learn_target_mounts(self):
for host, host_data in self.all_hosts_data.items():
# We will compare any found target mounts to all known MGSs
for local_info in host_data["local_targets"]:
debug_id = (host, local_info["device_paths"][0],
local_info["name"])
targets = ManagedTarget.objects.filter(uuid=local_info["uuid"])
if not targets.count():
log.warning("Ignoring %s:%s (%s), target unknown" %
debug_id)
continue
for target in targets:
if isinstance(target, FilesystemMember):
try:
mgs = self._target_find_mgs(host, local_info)
except ManagedMgs.DoesNotExist:
log.warning(
"Can't find MGS for target %s:%s (%s)" %
debug_id)
continue
else:
mgs = None
if not self.target_available_here(host, mgs, local_info):
log.warning(
"Ignoring %s on %s, as it is not mountable on this host"
% (local_info["name"], host))
continue
try:
log.info("Target %s seen on %s" % (target, host))
volumenode = self._get_volume_node(
host, local_info["device_paths"])
(tm,
created) = ManagedTargetMount.objects.get_or_create(
target=target, host=host, volume_node=volumenode)
if created:
if local_info["mounted"]:
tm.mount_point = local_info.get("mount_point")
tm.save()
log.info(
"Learned association %d between %s and host %s"
% (tm.id, local_info["name"], host))
self._learn_event(host, tm)
ObjectCache.add(ManagedTargetMount, tm)
if local_info["mounted"]:
target.state = "mounted"
target.active_mount = tm
label = local_info.get("ha_label")
if label:
target.ha_label = label
target.immutable_state = False
self.ha_targets[local_info["uuid"]] = {
"mount": tm.mount_point,
"paths": local_info["device_paths"],
}
target.save()
ObjectCache.update(target)
except NoNidsPresent:
log.warning(
"Cannot set up target %s on %s until LNet is running"
% (local_info["name"], host))
