def learn_fs_targets(self): for host, host_data in self.all_hosts_data.items(): for local_info in host_data["local_targets"]: if not local_info["mounted"]: log.warning("Ignoring unmounted target %s on host %s" % (local_info["name"], host)) continue name = local_info["name"] device_node_paths = local_info["device_paths"] uuid = local_info["uuid"] if name.find("-MDT") != -1: klass = ManagedMdt elif name.find("-OST") != -1: klass = ManagedOst elif name == "MGS": continue else: raise NotImplementedError() try: mgs = self._target_find_mgs(host, local_info) except ManagedMgs.DoesNotExist: log.warning("Can't find MGS for target %s on %s" % (name, host)) continue fsname, index_str = re.search("([\w\-]+)-(\w)+", name).groups() index = int(index_str, 16) # Create Filesystem objects if we've not seen this FS before. (filesystem, created) = ManagedFilesystem.objects.get_or_create( name=fsname, mgs=mgs) self.discovered_filesystems.add(filesystem) if created: self.created_filesystems.append(filesystem) filesystem.immutable_state = True filesystem.save() log.info("Found filesystem '%s'" % fsname) ObjectCache.add(ManagedFilesystem, filesystem) try: klass.objects.get(uuid=uuid) except ManagedTarget.DoesNotExist: # Fall through, no targets with that name exist on this MGS volumenode = self._get_volume_node(host, device_node_paths) target = klass( uuid=uuid, name=name, filesystem=filesystem, state="mounted", volume=volumenode.volume, index=index, immutable_state=True, ) target.save() log.debug("%s" % [mt.name for mt in ManagedTarget.objects.all()]) log.info("%s %s %s" % (mgs.id, name, device_node_paths)) log.info("Found %s %s" % (klass.__name__, name)) self.created_targets.append(target) ObjectCache.add(ManagedTarget, target.managedtarget_ptr)
def run(self, all_hosts_data): """:param all_hosts_data: Dict of ManagedHost to detect-scan output""" # Must be run in a transaction to avoid leaving invalid things # in the DB on failure. assert transaction.is_managed() self.all_hosts_data = all_hosts_data # Create ManagedMgs objects log.debug(">>learn_mgs_targets") self.learn_mgs_targets() # Create ManagedTargetMount objects log.debug(">>learn_mgs_target_mounts") self.learn_target_mounts() # Create ManagedMdt and ManagedOst objects log.debug(">>learn_fs_targets") self.learn_fs_targets() # Create ManagedTargetMount objects log.debug(">>learn_target_mounts") self.learn_target_mounts() # Assign a valid primary mount point, # and remove any targets which don't have a primary mount point for target in self.created_mgss + self.created_targets: if self.learn_primary_target(target): for tm in target.managedtargetmount_set.all(): self._learn_event(tm.host, target) else: self.log(help_text["found_no_primary_mount_point_for_target"] % (target.target_type(), target)) target.mark_deleted() if not self.created_filesystems: self.log(help_text["discovered_no_new_filesystem"]) else: # Remove any Filesystems with zero MDTs or zero OSTs, or set state # of a valid filesystem for fs in self.created_filesystems: mdt_count = ManagedMdt.objects.filter(filesystem=fs).count() ost_count = ManagedOst.objects.filter(filesystem=fs).count() if not mdt_count: self.log(help_text["found_not_TYPE_for_filesystem"] % ("MDT", fs.name)) fs.mark_deleted() elif not ost_count: self.log(help_text["found_not_TYPE_for_filesystem"] % ("OST", fs.name)) fs.mark_deleted() else: self.log(help_text[ "discovered_filesystem_with_n_MDTs_and_n_OSTs"] % (fs.name, mdt_count, ost_count)) if set([t.state for t in fs.get_targets()]) == set(['mounted']): fs.state = 'available' fs.save() first_target = fs.get_filesystem_targets()[0] self._learn_event(first_target.primary_host, first_target) if not self.created_mgss: self.log(help_text['discovered_no_new_target'] % ManagedMgs().target_type().upper()) else: for mgt in self.created_mgss: self.log( help_text['discovered_target'] % (mgt.target_type().upper(), mgt.name, mgt.primary_host)) ObjectCache.add(ManagedTarget, mgt.managedtarget_ptr) # Bit of additional complication so we can print really cracking messages, and detailed messages. for target in [ManagedMdt(), ManagedOst()]: if target.target_type() not in [ target.target_type() for target in self.created_targets ]: self.log(help_text['discovered_no_new_target'] % target.target_type().upper()) for target in self.created_targets: self.log(help_text['discovered_target'] % (target.target_type().upper(), target.name, target.primary_host))
def create_simple_filesystem(self, host): from chroma_core.models import ManagedMgs, ManagedMdt, ManagedOst, ManagedFilesystem, ManagedTargetMount self.mgt, _ = ManagedMgs.create_for_volume( synthetic_volume_full(host).id, name="MGS") self.fs = ManagedFilesystem.objects.create(mgs=self.mgt, name="testfs") ObjectCache.add(ManagedFilesystem, self.fs) ObjectCache.add(ManagedTarget, ManagedTarget.objects.get(id=self.mgt.id)) self.mdt, _ = ManagedMdt.create_for_volume( synthetic_volume_full(host).id, filesystem=self.fs) self.ost, _ = ManagedOst.create_for_volume( synthetic_volume_full(host).id, filesystem=self.fs) ObjectCache.add(ManagedTarget, ManagedTarget.objects.get(id=self.mdt.id)) ObjectCache.add(ManagedTarget, ManagedTarget.objects.get(id=self.ost.id)) ObjectCache.add(ManagedTargetMount, ManagedTargetMount.objects.get(target_id=self.mgt.id)) ObjectCache.add(ManagedTargetMount, ManagedTargetMount.objects.get(target_id=self.mdt.id)) ObjectCache.add(ManagedTargetMount, ManagedTargetMount.objects.get(target_id=self.ost.id))
def _create_simple_synthetic_object(class_, **kwargs): synthetic_object = class_(**kwargs) synthetic_object.save() ObjectCache.add(class_, synthetic_object) return synthetic_object
def create_simple_fs(fs_name="testfs"): # Create the MGT mgt = ManagedMgs.objects.create( id=1, state_modified_at="2020-11-11T23:52:23.938603+00:00", state="unmounted", immutable_state=False, name="MGS", uuid="uuid_mgt", ha_label=None, inode_size=None, bytes_per_inode=None, inode_count=None, reformat=False, not_deleted=True, ) mgt.save() ObjectCache.add(ManagedTarget, mgt.managedtarget_ptr) # Create a filesystem using the mgt fs = ManagedFilesystem.objects.create(mgs=mgt, name=fs_name, id=1, mdt_next_index=1, ost_next_index=1) ObjectCache.add(ManagedFilesystem, fs) # Create the mdt mdt = ManagedMdt.objects.create( id=2, index=0, filesystem_id=1, state_modified_at="2020-11-11T23:52:23.938603+00:00", state="unmounted", immutable_state=False, name="{}-MDT0000".format(fs_name), uuid="uuid_mdt", ha_label=None, inode_size=None, bytes_per_inode=None, inode_count=None, reformat=False, not_deleted=True, ) mdt.save() ObjectCache.add(ManagedTarget, mdt.managedtarget_ptr) # Create the ost ost = ManagedOst.objects.create( id=3, index=0, filesystem_id=1, state_modified_at="2020-11-11T23:52:23.938603+00:00", state="unmounted", immutable_state=False, name="{}-OST0000".format(fs_name), uuid="uuid_ost0", ha_label=None, inode_size=None, bytes_per_inode=None, inode_count=None, reformat=False, not_deleted=True, ) ost.save() ObjectCache.add(ManagedTarget, ost.managedtarget_ptr) return (mgt, fs, mdt, ost)
def learn_target_mounts(self): for host, host_data in self.all_hosts_data.items(): # We will compare any found target mounts to all known MGSs for local_info in host_data["local_targets"]: debug_id = (host, local_info["device_paths"][0], local_info["name"]) targets = ManagedTarget.objects.filter(uuid=local_info["uuid"]) if not targets.count(): log.warning("Ignoring %s:%s (%s), target unknown" % debug_id) continue for target in targets: if isinstance(target, FilesystemMember): try: mgs = self._target_find_mgs(host, local_info) except ManagedMgs.DoesNotExist: log.warning( "Can't find MGS for target %s:%s (%s)" % debug_id) continue else: mgs = None if not self.target_available_here(host, mgs, local_info): log.warning( "Ignoring %s on %s, as it is not mountable on this host" % (local_info["name"], host)) continue try: log.info("Target %s seen on %s" % (target, host)) volumenode = self._get_volume_node( host, local_info["device_paths"]) (tm, created) = ManagedTargetMount.objects.get_or_create( target=target, host=host, volume_node=volumenode) if created: if local_info["mounted"]: tm.mount_point = local_info.get("mount_point") tm.save() log.info( "Learned association %d between %s and host %s" % (tm.id, local_info["name"], host)) self._learn_event(host, tm) ObjectCache.add(ManagedTargetMount, tm) if local_info["mounted"]: target.state = "mounted" target.active_mount = tm label = local_info.get("ha_label") if label: target.ha_label = label target.immutable_state = False self.ha_targets[local_info["uuid"]] = { "mount": tm.mount_point, "paths": local_info["device_paths"], } target.save() ObjectCache.update(target) except NoNidsPresent: log.warning( "Cannot set up target %s on %s until LNet is running" % (local_info["name"], host))