Esempio n. 1
0
    def test_open_from_cache(self):
        # First, store example in cache
        fsconf, fs = create_lustrefs("../conf/models/example.lmf")
        self.assertTrue(fsconf)
        self.assertTrue(fs)

        # Try to re-read them
        fsconf, fs = open_lustrefs(fsconf.get_fs_name())
        self.assertTrue(fsconf)
        self.assertTrue(fs)

        fsconf.unregister_fs()
Esempio n. 2
0
    def test_open_from_cache(self):
        # First, store example in cache
        fsconf, fs = create_lustrefs("../conf/models/example.lmf")
        self.assertTrue(fsconf)
        self.assertTrue(fs)

        # Try to re-read them
        fsconf, fs = open_lustrefs(fsconf.get_fs_name())
        self.assertTrue(fsconf)
        self.assertTrue(fs)

        fsconf.unregister_fs()
Esempio n. 3
0
    def setUp(self):
        FSUtilsTest.setUp(self)

        tmpfile = makeTempFile("""
fs_name: param
nid_map: nodes=foo[1-10] nids=foo[1-10]@tcp
client: node=foo[7-10]
mgt: node=foo1 dev=/dev/sda
mdt: node=foo1 dev=/dev/sdb
ost: node=foo2 dev=/dev/sdc ha_node=foo3 ha_node=foo4 #index 0
ost: node=foo2 dev=/dev/sdd ha_node=foo4 ha_node=foo3 #index 1
ost: node=foo3 dev=/dev/sdc ha_node=foo2 ha_node=foo4 #index 2
ost: node=foo3 dev=/dev/sdd ha_node=foo4 ha_node=foo2 #index 3
router: node=foo[5-6]
mount_path: /param
        """)
        self.fsconf, self.fs = create_lustrefs(tmpfile.name)
Esempio n. 4
0
    def setUp(self):
        FSUtilsTest.setUp(self)

        tmpfile = makeTempFile("""
fs_name: param
nid_map: nodes=foo[1-13] nids=foo[1-13]@tcp
client: node=foo[8-11]
mgt: node=foo1 dev=/dev/sda
mdt: node=foo1 dev=/dev/sdb
ost: node=foo2 dev=/dev/sdc ha_node=foo3 ha_node=foo4 #index 0
ost: node=foo2 dev=/dev/sdd ha_node=foo4 ha_node=foo3 #index 1
ost: node=foo3 dev=/dev/sdc ha_node=foo2 ha_node=foo4 #index 2
ost: node=foo3 dev=/dev/sdd ha_node=foo4 ha_node=foo2 #index 3
ost: node=foo5 dev=/dev/sde ha_node=foo6 ha_node=foo7 #index 4
ost: node=foo5 dev=/dev/sdf ha_node=foo7 ha_node=foo6 #index 5
ost: node=foo6 dev=/dev/sde ha_node=foo5 ha_node=foo7 #index 6
ost: node=foo6 dev=/dev/sdf ha_node=foo7 ha_node=foo5 #index 7
router: node=foo[12-13]
mount_path: /param
        """)
        self.fsconf, self.fs = create_lustrefs(tmpfile.name)
Esempio n. 5
0
    def setUp(self):
        FSUtilsTest.setUp(self)

        tmpfile = makeTempFile(
            """
fs_name: param
nid_map: nodes=foo[1-13] nids=foo[1-13]@tcp
client: node=foo[8-11]
mgt: node=foo1 dev=/dev/sda
mdt: node=foo1 dev=/dev/sdb
ost: node=foo2 dev=/dev/sdc ha_node=foo3 ha_node=foo4 #index 0
ost: node=foo2 dev=/dev/sdd ha_node=foo4 ha_node=foo3 #index 1
ost: node=foo3 dev=/dev/sdc ha_node=foo2 ha_node=foo4 #index 2
ost: node=foo3 dev=/dev/sdd ha_node=foo4 ha_node=foo2 #index 3
ost: node=foo5 dev=/dev/sde ha_node=foo6 ha_node=foo7 #index 4
ost: node=foo5 dev=/dev/sdf ha_node=foo7 ha_node=foo6 #index 5
ost: node=foo6 dev=/dev/sde ha_node=foo5 ha_node=foo7 #index 6
ost: node=foo6 dev=/dev/sdf ha_node=foo7 ha_node=foo5 #index 7
router: node=foo[12-13]
mount_path: /param
        """
        )
        self.fsconf, self.fs = create_lustrefs(tmpfile.name)
Esempio n. 6
0
                        comp.tag, comp.TYPE.upper())
                ]
                if not self.options.dryrun:
                    oldconf.unregister_targets(tgtlist)

        #
        # NewFS
        #

        # Register the new conf
        self.__debug("Create new filesystem version")

        # XXX: Replace that with a simple fs save.
        newconf, newfs = create_lustrefs(lmf,
                                         nodes=self.options.nodes,
                                         excluded=self.options.excludes,
                                         event_handler=neweh,
                                         update_mode=True)
        newfs.set_debug(self.options.debug)

        # Register the new targets in backend
        if 'format' in actions:
            self.__verbose("Register target(s) %s into backend." %
                           actions['format'].labels())
            for comp in actions['format']:
                tgtlist = [
                    newconf.get_target_from_tag_and_type(
                        comp.tag, comp.TYPE.upper())
                ]
                if not self.options.dryrun:
                    oldconf.register_targets(tgtlist)
Esempio n. 7
0
 def test_create_example(self):
     fsconf, fs = create_lustrefs("../conf/models/example.lmf")
     self.assertTrue(fsconf)
     self.assertTrue(fs)
     fsconf.unregister_fs()
Esempio n. 8
0
    def execute(self):

        # Option sanity check
        self.forbidden(self.options.fsnames, "-f, see -m")
        self.forbidden(self.options.labels, "-l")
        self.forbidden(self.options.indexes, "-i")
        self.forbidden(self.options.failover, "-F")

        rc = RC_OK

        if not self.options.model:
            raise CommandHelpException(
                "Lustre model file path"
                "(-m <model_file>) argument required.", self)

        eh = FSGlobalEventHandler(self)

        # Use this Shine.FSUtils convenience function.
        lmf = self.get_lmf_path()
        if lmf:
            print "Using Lustre model file %s" % lmf
        else:
            raise CommandHelpException(
                "Lustre model file for ``%s'' not found:"
                " please use filename or full LMF path.\n"
                "Your default model files directory (lmf_dir) is: %s" %
                (self.options.model, Globals().get_lmf_dir()), self)

        install_nodes = self.options.nodes
        excluded_nodes = self.options.excludes

        fs_conf, fs = create_lustrefs(self.get_lmf_path(),
                                      event_handler=eh,
                                      nodes=install_nodes,
                                      excluded=excluded_nodes)

        # Register the filesystem in backend
        print "Registering FS %s to backend..." % fs.fs_name
        if self.options.dryrun:
            rc = 0
        else:
            rc = self.register_fs(fs_conf)
        if rc:
            msg = "Error: failed to register FS to backend (rc=%d)" % rc
            print >> sys.stderr, msg
        else:
            print "Filesystem %s registered." % fs.fs_name

        # Helper message.
        # If user specified nodes which were not used, warn him about it.
        actual_nodes = fs.components.managed().servers()
        if not self.check_valid_list(fs_conf.get_fs_name(), \
                actual_nodes, "install"):
            return RC_FAILURE

        # Install file system configuration files; normally, this should
        # not be done by the Shine.Lustre.FileSystem object itself, but as
        # all proxy methods are currently handled by it, it is more
        # convenient this way...
        try:
            fs.install(fs_conf.get_cfg_filename(), dryrun=self.options.dryrun)

            tuning_conf = Globals().get_tuning_file()
            if tuning_conf:
                fs.install(tuning_conf, dryrun=self.options.dryrun)

        except FSRemoteError, error:
            print "WARNING: Due to error, installation skipped on %s" \
                   % error.nodes
            rc = RC_FAILURE
Esempio n. 9
0
    def execute(self):

        # Option sanity check
        self.forbidden(self.options.fsnames, "-f, see -m")
        self.forbidden(self.options.labels, "-l")
        self.forbidden(self.options.indexes, "-i")
        self.forbidden(self.options.failover, "-F")

        rc = RC_OK

        # Check
        lmf = self.lmfpath()

        # Load next model
        newconf = open_model(lmf)
        newfsconf = newconf._fs
        newfsconf.setup_target_devices(update_mode=True)
        neweh = self.GLOBAL_EH(self)
        newfs = instantiate_lustrefs(newconf,
                                     nodes=self.options.nodes,
                                     excluded=self.options.excludes,
                                     event_handler=neweh)
        newfs.set_debug(self.options.debug)

        # Load current registered FS
        oldeh = self.GLOBAL_EH(self)
        oldconf, oldfs = open_lustrefs(newfsconf.fs_name,
                                       nodes=self.options.nodes,
                                       excluded=self.options.excludes,
                                       event_handler=oldeh)
        oldfs.set_debug(self.options.debug)

        # Compare them
        actions = oldconf._fs.compare(newfsconf)

        # Convert Configuration objects to ComponentGroup
        # for old filesystem
        oldcomps = ComponentGroup()
        for action in ('unmount', 'stop', 'remove'):
            if action in actions:
                actions[action] = convert_comparison(
                    oldconf, oldfs, actions[action]).managed()
                if len(actions[action]) == 0:
                    del actions[action]
                else:
                    for comp in actions[action]:
                        if comp not in oldcomps:
                            oldcomps.add(comp)
        # for new filesystem
        for action in ('format', 'start', 'mount'):
            if action in actions:
                # XXX: Do we need to add .managed() here?
                actions[action] = convert_comparison(newconf, newfs,
                                                     actions[action])

        self.display_changes(actions)

        # XXX: Update message with node list
        if not self.ask_confirm(
                "Update `%s': do you want to continue?" % oldfs.fs_name):
            return RC_FAILURE

        # Will call the handle_pre() method defined by the event handler.
        if hasattr(oldeh, 'pre'):
            oldeh.pre(oldfs)

        #
        # UNINSTALL unused component for old filesystem version.
        #

        try:

            # Check status of removed components
            if len(oldcomps):
                self._precheck(oldfs, oldfs.status, 'verify', comps=oldcomps)

            # Unmount what will be removed or remounted
            if 'unmount' in actions:
                comps = actions['unmount']
                self._apply(oldfs, oldfs.umount, 'unmount', comps, OFFLINE)

            # Stop what need to be stopped or will be removed
            if 'stop' in actions:
                self._apply(oldfs, oldfs.stop, 'stop', actions['stop'],
                            OFFLINE)

            # Remove conf on now unused nodes
            # XXX: This does not take _precheck() status into account.
            oldservers = oldcomps.managed().allservers()
            newservers = newfs.components.managed().allservers()
            removedsrvs = oldservers.difference(newservers)
            if len(removedsrvs) > 0:
                self.__verbose("Remove configuration from %s" % removedsrvs)
                self._remove(oldfs, oldfs.remove, "uninstall", removedsrvs)

        except CannotApplyError as exp:
            self.__warning(str(exp))
            print("Please fix the error or disable %s and restart the update" %
                  exp.elements + " command")
            return 1

        # Unregister from backend
        if 'remove' in actions:
            self.__verbose("Remove target(s) %s from backend." %
                           actions['remove'].labels())
            for comp in actions['remove'].filter(supports='dev'):
                tgtlist = [
                    oldconf.get_target_from_tag_and_type(
                        comp.tag, comp.TYPE.upper())
                ]
                if not self.options.dryrun:
                    oldconf.unregister_targets(tgtlist)

        #
        # NewFS
        #

        # Register the new conf
        self.__debug("Create new filesystem version")

        # XXX: Replace that with a simple fs save.
        newconf, newfs = create_lustrefs(lmf,
                                         nodes=self.options.nodes,
                                         excluded=self.options.excludes,
                                         event_handler=neweh,
                                         update_mode=True)
        newfs.set_debug(self.options.debug)

        # Register the new targets in backend
        if 'format' in actions:
            self.__verbose("Register target(s) %s into backend." %
                           actions['format'].labels())
            for comp in actions['format']:
                tgtlist = [
                    newconf.get_target_from_tag_and_type(
                        comp.tag, comp.TYPE.upper())
                ]
                if not self.options.dryrun:
                    oldconf.register_targets(tgtlist)

        # Will call the handle_pre() method defined by the event handler.
        if hasattr(neweh, 'pre'):
            neweh.pre(newfs)

        # Update with new conf
        # Note: For user convenience, we always copy configuration, this could
        # help when nodes are misinstalled.
        self._copy(newfs, newconf.get_cfg_filename())
        if Globals().get_tuning_file():
            self._copy(newfs, Globals().get_tuning_file())

        next_actions = []

        # Tunefs if needed
        if 'tunefs' in actions or 'writeconf' in actions:
            next_actions.append("Need to run `tunefs' on all components.")
            next_actions.append(self._next_action_cmd('tunefs', newfs))

        # Reformat if needed
        if 'reformat' in actions:
            next_actions.append("Need to `reformat' all targets.")
            next_actions.append(self._next_action_cmd('format', newfs))

        # Format if needed
        if 'format' in actions:
            # XXX: Check if everything is already stopped?
            next_actions.append("You can now `format' %d new target(s)" % \
                                len(actions['format']))
            next_actions.append(
                self._next_action_cmd('format', newfs,
                                      '-l %s' % actions['format'].labels()))
        # Start if needed
        if 'start' in actions:
            next_actions.append("You can now `start' %d new component(s)" % \
                                len(actions['start']))
            next_actions.append(
                self._next_action_cmd('start', newfs,
                                      '-l %s' % actions['start'].labels()))

        # Mount if needed
        if 'mount' in actions:
            next_actions.append("You can now `mount' the needed %d client(s)" %
                                len(actions['mount']))
            next_actions.append(
                self._next_action_cmd('mount', newfs,
                                      '-n %s' % actions['mount'].servers()))

        # Tune if needed
        if 'tune' in actions:
            next_actions.append("Need tu run `tune' on all components.")
            next_actions.append(self._next_action_cmd('tune', newfs))

        # Print this line only if there is other actions to be performed
        if next_actions:
            print()
            print("NEXT ACTIONS (should be done manually)")
            for txt in next_actions:
                print(">%s" % txt)

        print("Update is finished.")

        return rc
Esempio n. 10
0
    def execute(self):

        # Option sanity check
        self.forbidden(self.options.fsnames, "-f, see -m")
        self.forbidden(self.options.labels, "-l")
        self.forbidden(self.options.indexes, "-i")
        self.forbidden(self.options.failover, "-F")

        rc = RC_OK

        if not self.options.model:
            raise CommandHelpException("Lustre model file path" "(-m <model_file>) argument required.", self)

        eh = FSGlobalEventHandler(self)

        # Use this Shine.FSUtils convenience function.
        lmf = self.get_lmf_path()
        if lmf:
            print "Using Lustre model file %s" % lmf
        else:
            raise CommandHelpException(
                "Lustre model file for ``%s'' not found:"
                " please use filename or full LMF path.\n"
                "Your default model files directory (lmf_dir) is: %s" % (self.options.model, Globals().get_lmf_dir()),
                self,
            )

        install_nodes = self.options.nodes
        excluded_nodes = self.options.excludes

        fs_conf, fs = create_lustrefs(
            self.get_lmf_path(), event_handler=eh, nodes=install_nodes, excluded=excluded_nodes
        )

        # Register the filesystem in backend
        print "Registering FS %s to backend..." % fs.fs_name
        if self.options.dryrun:
            rc = 0
        else:
            rc = self.register_fs(fs_conf)
        if rc:
            msg = "Error: failed to register FS to backend (rc=%d)" % rc
            print >> sys.stderr, msg
        else:
            print "Filesystem %s registered." % fs.fs_name

        # Helper message.
        # If user specified nodes which were not used, warn him about it.
        actual_nodes = fs.components.managed().servers()
        if not self.check_valid_list(fs_conf.get_fs_name(), actual_nodes, "install"):
            return RC_FAILURE

        # Install file system configuration files; normally, this should
        # not be done by the Shine.Lustre.FileSystem object itself, but as
        # all proxy methods are currently handled by it, it is more
        # convenient this way...
        try:
            fs.install(fs_conf.get_cfg_filename(), dryrun=self.options.dryrun)

            tuning_conf = Globals().get_tuning_file()
            if tuning_conf:
                fs.install(tuning_conf, dryrun=self.options.dryrun)

        except FSRemoteError, error:
            print "WARNING: Due to error, installation skipped on %s" % error.nodes
            rc = RC_FAILURE
Esempio n. 11
0
            for comp in actions['stop'].filter(supports='dev'):
                tgtlist = [oldconf.get_target_from_tag_and_type(
                                 comp.tag, comp.TYPE.upper())]
                oldconf.unregister_targets(tgtlist)

        #
        # NewFS
        #

        # Register the new conf
        self.__debug("Create new filesystem version")

        # XXX: Replace that with a simple fs save.
        newconf, newfs = create_lustrefs(lmf, 
                                         nodes=self.options.nodes,
                                         excluded=self.options.excludes,
                                         event_handler=neweh,
                                         update_mode=True)
        newfs.set_debug(self.options.debug)

        # Register the new targets in backend
        if 'format' in actions:
            self.__verbose("Register target(s) %s into backend." %
                           actions['format'].labels())
            for comp in actions['format']:
                tgtlist = [newconf.get_target_from_tag_and_type(comp.tag,
                                                            comp.TYPE.upper())]
                oldconf.register_targets(tgtlist)

        # Will call the handle_pre() method defined by the event handler.
        if hasattr(neweh, 'pre'):
Esempio n. 12
0
 def test_create_example(self):
     fsconf, fs = create_lustrefs("../conf/models/example.lmf")
     self.assertTrue(fsconf)
     self.assertTrue(fs)
     fsconf.unregister_fs()