Esempio n. 1
0
 def test_big_nid_map_scalable(self):
     """check big nid mapping is scalable."""
     model = Model()
     model.parse("nid_map: nodes=foo[1-9999] nids=bar[1-9999]@tcp")
     before = time.time()
     NidMap.fromlist(model.get('nid_map'))
     self.assertTrue(time.time() - before < .5)
Esempio n. 2
0
 def test_big_nid_map_scalable(self):
     """check big nid mapping is scalable."""
     model = Model()
     model.parse("nid_map: nodes=foo[1-9999] nids=bar[1-9999]@tcp")
     before = time.time()
     NidMap.fromlist(model.get("nid_map"))
     self.assertTrue(time.time() - before < 0.5)
Esempio n. 3
0
    def test_match_device_missing_prop(self):
        """check missing property does not match in match_device()"""
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
mgt: node=foo7 network=tcp""")
        candidate = [{'node': 'foo7', 'dev': '/dev/sda'}]
        self.assertEqual(len(model.get('mgt')[0].match_device(candidate)), 0)
Esempio n. 4
0
    def test_match_device_error(self):
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
mgt: node=(\<badregexp>""")
        candidate = [ {'node': 'foo7', 'dev': '/dev/sda'} ]
        self.assertRaises(ModelFileValueError, model.get('mgt')[0].match_device,
                candidate)
Esempio n. 5
0
    def test_match_device_missing_prop(self):
        """check missing property does not match in match_device()"""
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
mgt: node=foo7 network=tcp""")
        candidate = [ {'node': 'foo7', 'dev': '/dev/sda'} ]
        self.assertEqual(len(model.get('mgt')[0].match_device(candidate)), 0)
Esempio n. 6
0
    def test_match_device_error(self):
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
mgt: node=(\<badregexp>""")
        candidate = [{'node': 'foo7', 'dev': '/dev/sda'}]
        self.assertRaises(ModelFileValueError,
                          model.get('mgt')[0].match_device, candidate)
Esempio n. 7
0
 def test_match_device_simple_pattern(self):
     """check different values do not match in match_device()"""
     candidate = [{'node': 'foo7', 'dev': '/dev/sda'}]
     model = Model()
     model.parse(
         textwrap.dedent("""fs_name: match
         nid_map: nodes=foo8 nids=foo8@tcp
         mgt: node=foo8"""))
     self.assertEqual(len(model.get('mgt')[0].match_device(candidate)), 0)
Esempio n. 8
0
    def test_match_device_ignore_index(self):
        """test match_device() does not try to match index property"""
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
ost: node=foo7 index=0""")
        candidate = [{'node': 'foo7', 'dev': '/dev/sda'}]
        matched = model.get('ost')[0].match_device(candidate)
        self.assertEqual(len(matched), 1)
        self.assertEqual(matched[0].get('dev'), '/dev/sda')
Esempio n. 9
0
    def test_match_device_ignore_index(self):
        """test match_device() does not try to match index property"""
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[7] nids=foo[7]@tcp
ost: node=foo7 index=0""")
        candidate = [ {'node': 'foo7', 'dev': '/dev/sda'} ]
        matched = model.get('ost')[0].match_device(candidate)
        self.assertEqual(len(matched), 1)
        self.assertEqual(matched[0].get('dev'), '/dev/sda')
Esempio n. 10
0
    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Set nodes to nids mapping using the NidMap helper class
        self.nid_map = NidMap.fromlist(self.get('nid_map'))

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()
Esempio n. 11
0
 def test_folding(self):
     """config lines are grouped when possible"""
     model = Model()
     model.parse(
         textwrap.dedent("""
         fs_name: fold
         nid_map: nodes=foo[1-2] nids=foo[1-2]@tcp2
         client: node=foo1
         client: node=foo2
         router: node=foo1
         router: node=foo2"""))
     self.assertEqual(
         str(model),
         textwrap.dedent("""
         fs_name:fold
         client:node=foo[1-2]
         router:node=foo[1-2]
         nid_map:nids=foo[1-2]@tcp2 nodes=foo[1-2]""").lstrip())
Esempio n. 12
0
    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Model expands nid_map automatically, just iterate other them
        self.nid_map = {}
        for elem in self.get('nid_map'):
            self.nid_map.setdefault(elem['nodes'], []).append(elem['nids'])

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()
Esempio n. 13
0
    def test_diff_client(self):
        """check diff detects updated clients."""
        model = Model()
        model.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
client: node=foo1""")

        new_m = Model()
        new_m.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
client: node=foo1 mount_options=ro""")

        added, changed, removed = model.diff(new_m)
        self.assertEqual(len(added), 0)
        self.assertEqual(len(removed), 0)
        self.assertEqual(str(changed), "client:node=foo1 mount_options=ro")
Esempio n. 14
0
    def test_diff_ost(self):
        """check diff detects updated targets."""
        model = Model()
        model.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
ost: node=foo1 dev=/dev/sda
ost: node=foo1 dev=/dev/sdc mode=external """)

        new_m = Model()
        new_m.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
ost: node=foo1 dev=/dev/sdc jdev=/dev/sdd """)

        added, changed, removed = model.diff(new_m)
        self.assertEqual(len(added), 0)
        self.assertEqual(str(removed), "ost:node=foo1 dev=/dev/sda")
        self.assertEqual(str(changed),
                         "ost:node=foo1 dev=/dev/sdc jdev=/dev/sdd")
Esempio n. 15
0
    def test_several_spaces(self):
        model = Model()
        model.parse("""fs_name:  spaces 
nid_map:  nodes=foo[7]  nids=foo[7]@tcp
mgt:  node=foo7 """)
        self.assertEqual(model.get('fs_name'), 'spaces')
        self.assertEqual(len(model.elements('nid_map')), 1)
        self.assertEqual(
            model.elements('nid_map')[0].as_dict(), {
                'nodes': 'foo[7]',
                'nids': 'foo[7]@tcp'
            })
Esempio n. 16
0
    def test_match_device_list_values(self):
        """check various list content do not match in match_device()"""
        # Different value count
        candidate = [{'node': 'foo7', 'ha_node': ['foo8'], 'dev': '/dev/sda'}]
        model = Model()
        model.parse(
            textwrap.dedent("""fs_name: nids
            nid_map: nodes=foo[7-10] nids=foo[7-10]@tcp
            mgt: node=foo7 ha_node=foo8 ha_node=foo9"""))
        self.assertEqual(len(model.get('mgt')[0].match_device(candidate)), 0)

        # Different list values
        candidate = [{
            'node': 'foo7',
            'ha_node': ['foo7', 'foo8'],
            'dev': '/dev/sda'
        }]
        model = Model()
        model.parse(
            textwrap.dedent("""fs_name: nids
            nid_map: nodes=foo[7-10] nids=foo[7-10]@tcp
            mgt: node=foo7 ha_node=foo8 ha_node=foo9"""))
        self.assertEqual(len(model.get('mgt')[0].match_device(candidate)), 0)
Esempio n. 17
0
    def test_several_spaces(self):
        model = Model()
        model.parse("""fs_name:  spaces 
nid_map:  nodes=foo[7]  nids=foo[7]@tcp
mgt:  node=foo7 """)
        self.assertEqual(model.get('fs_name'), 'spaces')
        self.assertEqual(len(model.elements('nid_map')), 1)
        self.assertEqual(model.elements('nid_map')[0].as_dict(),
                { 'nodes': 'foo[7]', 'nids': 'foo[7]@tcp' })
Esempio n. 18
0
    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Set nodes to nids mapping using the NidMap helper class
        self.nid_map = NidMap.fromlist(self.get('nid_map'))

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()
Esempio n. 19
0
    def test_diff_client(self):
        """check diff detects updated clients."""
        model = Model()
        model.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
client: node=foo1""")

        new_m = Model()
        new_m.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
client: node=foo1 mount_options=ro""")

        added, changed, removed = model.diff(new_m)
        self.assertEqual(len(added), 0)
        self.assertEqual(len(removed), 0)
        self.assertEqual(str(changed), "client:node=foo1 mount_options=ro")
Esempio n. 20
0
 def test_2_nid_map_diff_pattern(self):
     """Model with nid_map with several ranges"""
     model = Model()
     model.parse(
         textwrap.dedent("""fs_name: nids
         nid_map: nodes=foo[1-2] nids=foo[1-2]@tcp
         nid_map: nodes=bar[1-9] nids=bar[1-9]@tcp"""))
     self.assertEqual(len(model.elements('nid_map')), 11)
     self.assertEqual(
         model.elements('nid_map')[0].as_dict(), {
             'nodes': 'foo1',
             'nids': 'foo1@tcp'
         })
     self.assertEqual(
         model.elements('nid_map')[10].as_dict(), {
             'nodes': 'bar9',
             'nids': 'bar9@tcp'
         })
Esempio n. 21
0
    def test_diff_ost(self):
        """check diff detects updated targets."""
        model = Model()
        model.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
ost: node=foo1 dev=/dev/sda
ost: node=foo1 dev=/dev/sdc mode=external """)

        new_m = Model()
        new_m.parse("""fs_name: diff
nid_map: nodes=foo1 nids=foo1@tcp
ost: node=foo1 dev=/dev/sdc jdev=/dev/sdd """)

        added, changed, removed = model.diff(new_m)
        self.assertEqual(len(added), 0)
        self.assertEqual(str(removed), "ost:node=foo1 dev=/dev/sda")
        self.assertEqual(str(changed),
                                    "ost:node=foo1 dev=/dev/sdc jdev=/dev/sdd")
Esempio n. 22
0
    def test_several_nid_map(self):
        """Model with several nid_map lines."""
        model = Model()
        model.parse("""fs_name: nids
nid_map: nodes=foo[1-2] nids=foo[1-2]@tcp
nid_map: nodes=foo[7] nids=foo[7]@tcp""")
        self.assertEqual(len(model.elements('nid_map')), 3)
        self.assertEqual(
            model.elements('nid_map')[0].as_dict(), {
                'nodes': 'foo1',
                'nids': 'foo1@tcp'
            })
        self.assertEqual(
            model.elements('nid_map')[1].as_dict(), {
                'nodes': 'foo2',
                'nids': 'foo2@tcp'
            })
        self.assertEqual(
            model.elements('nid_map')[2].as_dict(), {
                'nodes': 'foo7',
                'nids': 'foo7@tcp'
            })
Esempio n. 23
0
 def makeTempModel(self, txt):
     """helper method for creating a temp file and loading it as Model"""
     self._testfile = makeTempFile(txt)
     model = Model()
     model.load(self._testfile.name)
     return model
Esempio n. 24
0
class FileSystem(object):
    """
    Lustre File System Configuration class.
    """
    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Set nodes to nids mapping using the NidMap helper class
        self.nid_map = NidMap.fromlist(self.get('nid_map'))

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()

    @property
    def fs_name(self):
        return self.get('fs_name')

    def get(self, key, default=None):
        """Return the Model value pointed by `key'"""
        return self.model.get(key, default)

    @classmethod
    def _cache_path(cls, fsname):
        """Build and check a cache file path from filesystem name."""
        fs_conf_dir = os.path.expandvars(Globals().get_conf_dir())
        if not os.path.exists(fs_conf_dir):
            raise ConfigException("Cache directory does not exist '%s'" %
                                  fs_conf_dir)
        return "%s/%s.xmf" % (os.path.normpath(fs_conf_dir), fsname)

    @classmethod
    def create_from_model(cls, lmf, update_mode=False):
        """Save to cache."""
        fsmodel = FileSystem(lmf)
        # xmf_path could be set later if setup_target_devices do not need it
        fsmodel.xmf_path = cls._cache_path(fsmodel.fs_name)
        fsmodel.setup_target_devices(update_mode=update_mode)
        # Save XMF
        fsmodel.model.save(fsmodel.xmf_path,
                  "# Shine Lustre file system config file for %s" % \
                  fsmodel.fs_name)

        # Reload from content saved previously
        return cls.load_from_fsname(fsmodel.fs_name)

    @classmethod
    def load_from_fsname(cls, fsname):
        """Load from cache."""
        conf_file = cls._cache_path(fsname)
        fsconf = FileSystem(conf_file)
        fsconf.xmf_path = conf_file
        return fsconf

    def _start_backend(self):
        """
        Load and start backend subsystem once
        """
        if not self.backend:

            # Start the selected config backend system.
            self.backend = BackendRegistry().selected()
            if self.backend:
                self.backend.start()

        return self.backend

    def setup_target_devices(self, update_mode=False):
        """ Generate the eXtended Model File XMF """

        self._start_backend()

        # We have to setup the possible targets, which are: MGT, MDT and OST.
        for target in ['mgt', 'mdt', 'ost']:

            # So, first, look for which ones are defined in model
            if target not in self.model:
                continue

            # Lustre supports up to FFFF targets per type.
            indexes = range(0, 65535)

            if self.backend:

                # Returns a list of TargetDevices
                fs_name = None
                if update_mode == True:
                    fs_name = self.fs_name
                candidates = self.backend.get_target_devices(
                    target, fs_name=fs_name, update_mode=update_mode)

                # Save the model target selection
                target_models = copy.copy(self.model.get(target))

                # Reduce entropy from backend.
                # (node, dev) should point to a unique device, so this should
                # be enough for sorting. This is not perfect but enough to have
                # a very low entropy in result order.
                candidates.sort(key=lambda x: (x.get('node'), x.get('dev')))

                # Delete it (to be replaced... see below)
                self.model.elements(target).clear()

                try:

                    # Remove already used index from candidate list.
                    for target_model in target_models:
                        if 'index' in target_model:
                            idx = target_model.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Iterates on each Model.Target
                    for target_model in target_models:

                        # Do not try to match external components
                        if target_model.get('mode') == 'external':
                            self.model.elements(target).parse(
                                str(target_model))
                            continue

                        result = target_model.match_device(candidates)
                        if len(result) == 0:
                            raise ConfigDeviceNotFoundError(target_model)

                        for matching in result:
                            candidates.remove(matching)

                            # If an index was specified, set it.
                            if 'index' in target_model:
                                matching.add_index(target_model.get('index'))
                                target_model.elements('index').clear()

                            # Manage index, mandatoy in XMF files
                            if not matching.has_index():
                                matching.add_index(indexes[0])
                                idx = matching.index()
                                # List raises ValueError and we need to know
                                # the missing indexes here, so we had this
                                # hack.
                                if idx not in indexes:
                                    raise KeyError(idx)
                                indexes.remove(idx)

                            # Copy properties from model
                            # which do not exist in backend.
                            matching.add_active(target_model.get('active'))

                            # `matching' is a TargetDevice, we want to add it
                            # to the underlying Model object. The current way
                            # to do this to create a configuration line string
                            # (performed by TargetDevice.getline()) and then
                            # call Model.parse().
                            # TODO: add methods to Model/ModelDevice to avoid
                            #  the use of temporary configuration string line.
                            self.model.elements(target).parse(
                                matching.getline())

                except KeyError, error:
                    raise ConfigInvalidFileSystem(self, \
                            "Index %s for %s used twice." % \
                            (str(error), target))

            # Support for backend None
            else:

                try:
                    # Remove already used indexes from candidate list.
                    for params in self.model.elements(target):
                        if 'index' in params:
                            idx = params.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Manage index
                    for params in self.model.elements(target):
                        if 'index' not in params:
                            params.add('index', str(indexes[0]))
                            indexes.remove(indexes[0])

                except KeyError, error:
                    raise ConfigInvalidFileSystem(self, \
                             "Index %s for %s used twice." % \
                              (str(error), target))
Esempio n. 25
0
 def testDefaultValues(self):
     """test defaults values"""
     m = Model()
     self.assertEqual(m.get("stripe_size"), 1048576)
     self.assertEqual(m.get("stripe_count"), 1)
     self.assertEqual(m.get("quota_type"), "ug")
Esempio n. 26
0
 def testLoadExample(self):
     """Load example.lmf and checks it."""
     m = Model()
     m.load("../conf/models/example.lmf")
     self.assertEqual(len(m), 19)
Esempio n. 27
0
 def testDefaultValues(self):
     """test defaults values"""
     m = Model()
     self.assertEqual(m.get('stripe_size'), 1048576)
     self.assertEqual(m.get('stripe_count'), 1)
     self.assertEqual(m.get('quota_type'), 'ug')
Esempio n. 28
0
 def makeTempModel(self, txt):
     """helper method for creating a temp file and loading it as Model"""
     self._testfile = makeTempFile(txt)
     model = Model()
     model.load(self._testfile.name)
     return model
Esempio n. 29
0
 def testLoadExample(self):
     """Load example.lmf and checks it."""
     m = Model()
     m.load('../conf/models/example.lmf')
     self.assertEqual(len(m), 15)
Esempio n. 30
0
class FileSystem(object):
    """
    Lustre File System Configuration class.
    """

    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Set nodes to nids mapping using the NidMap helper class
        self.nid_map = NidMap.fromlist(self.get('nid_map'))

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()

    @property
    def fs_name(self):
        return self.get('fs_name')

    def get(self, key, default=None):
        """Return the Model value pointed by `key'"""
        return self.model.get(key, default)

    @classmethod
    def _cache_path(cls, fsname):
        """Build and check a cache file path from filesystem name."""
        fs_conf_dir = os.path.expandvars(Globals().get_conf_dir())
        if not os.path.exists(fs_conf_dir):
            raise ConfigException("Cache directory does not exist '%s'" % 
                                  fs_conf_dir)
        return "%s/%s.xmf" % (os.path.normpath(fs_conf_dir), fsname)

    @classmethod
    def create_from_model(cls, lmf, update_mode=False):
        """Save to cache."""
        fsmodel = FileSystem(lmf)
        # xmf_path could be set later if setup_target_devices do not need it
        fsmodel.xmf_path = cls._cache_path(fsmodel.fs_name)
        fsmodel.setup_target_devices(update_mode=update_mode)
        # Save XMF
        fsmodel.model.save(fsmodel.xmf_path,
                  "# Shine Lustre file system config file for %s" % \
                  fsmodel.fs_name)

        # Reload from content saved previously
        return cls.load_from_fsname(fsmodel.fs_name)

    @classmethod
    def load_from_fsname(cls, fsname):
        """Load from cache."""
        conf_file = cls._cache_path(fsname)
        fsconf = FileSystem(conf_file)
        fsconf.xmf_path = conf_file
        return fsconf


    def _start_backend(self):
        """
        Load and start backend subsystem once
        """
        if not self.backend:

            # Start the selected config backend system.
            self.backend = BackendRegistry().selected()
            if self.backend:
                self.backend.start()

        return self.backend

    def setup_target_devices(self, update_mode=False):
        """ Generate the eXtended Model File XMF """

        self._start_backend()

        # We have to setup the possible targets, which are: MGT, MDT and OST.
        for target in [ 'mgt', 'mdt', 'ost' ]:

            # So, first, look for which ones are defined in model
            if target not in self.model:
                continue

            # Lustre supports up to FFFF targets per type.
            indexes = range(0, 65535)

            if self.backend:

                # Returns a list of TargetDevices
                fs_name = None
                if update_mode == True:
                    fs_name = self.fs_name
                candidates = self.backend.get_target_devices(target,
                                      fs_name=fs_name, update_mode=update_mode)

                # Save the model target selection
                target_models = copy.copy(self.model.get(target))

                # Reduce entropy from backend.
                # (node, dev) should point to a unique device, so this should
                # be enough for sorting. This is not perfect but enough to have
                # a very low entropy in result order.
                candidates.sort(key=lambda x: (x.get('node'), x.get('dev')))

                # Delete it (to be replaced... see below)
                self.model.elements(target).clear()

                try:

                    # Remove already used index from candidate list.
                    for target_model in target_models:
                        if 'index' in target_model:
                            idx = target_model.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Iterates on each Model.Target
                    for target_model in target_models:

                        # Do not try to match external components
                        if target_model.get('mode') == 'external':
                            self.model.elements(target).parse(str(target_model))
                            continue

                        result = target_model.match_device(candidates)
                        if len(result) == 0:
                            raise ConfigDeviceNotFoundError(target_model)

                        for matching in result:
                            candidates.remove(matching)

                            # If an index was specified, set it.
                            if 'index' in target_model:
                                matching.add_index(target_model.get('index'))
                                target_model.elements('index').clear()

                            # Manage index, mandatoy in XMF files
                            if not matching.has_index():
                                matching.add_index(indexes[0])
                                idx = matching.index()
                                # List raises ValueError and we need to know
                                # the missing indexes here, so we had this
                                # hack.
                                if idx not in indexes:
                                    raise KeyError(idx)
                                indexes.remove(idx)

                            # Copy properties from model
                            # which do not exist in backend.
                            matching.add_active(target_model.get('active'))

                            # `matching' is a TargetDevice, we want to add it
                            # to the underlying Model object. The current way
                            # to do this to create a configuration line string
                            # (performed by TargetDevice.getline()) and then
                            # call Model.parse(). 
                            # TODO: add methods to Model/ModelDevice to avoid
                            #  the use of temporary configuration string line.
                            self.model.elements(target).parse(
                                                            matching.getline())

                except KeyError, error:
                    raise ConfigInvalidFileSystem(self, \
                            "Index %s for %s used twice." % \
                            (str(error), target))

            # Support for backend None
            else:

                try:
                    # Remove already used indexes from candidate list.
                    for params in self.model.elements(target):
                        if 'index' in params:
                            idx = params.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Manage index
                    for params in self.model.elements(target):
                        if 'index' not in params:
                            params.add('index', str(indexes[0]))
                            indexes.remove(indexes[0])

                except KeyError, error:
                    raise ConfigInvalidFileSystem(self, \
                             "Index %s for %s used twice." % \
                              (str(error), target))
Esempio n. 31
0
class FileSystem(object):
    """
    Lustre File System Configuration class.
    """
    def __init__(self, filename):

        self.backend = None
        self.xmf_path = None
        self.model = Model()

        try:
            self.model.load(filename)
        except IOError:
            raise ModelFileIOError("Could not read %s" % filename)

        # Model expands nid_map automatically, just iterate other them
        self.nid_map = {}
        for elem in self.get('nid_map'):
            self.nid_map.setdefault(elem['nodes'], []).append(elem['nids'])

        # Initialize the tuning model to None if no special tuning configuration
        # is provided
        self.tuning_model = TuningModel()

    @property
    def fs_name(self):
        return self.get('fs_name')

    def get(self, key, default=None):
        """Return the Model value pointed by `key'"""
        return self.model.get(key, default)

    @classmethod
    def _cache_path(cls, fsname):
        """Build and check a cache file path from filesystem name."""
        fs_conf_dir = os.path.expandvars(Globals().get_conf_dir())
        if not os.path.exists(fs_conf_dir):
            raise ConfigException("Cache directory does not exist '%s'" %
                                  fs_conf_dir)
        return "%s/%s.xmf" % (os.path.normpath(fs_conf_dir), fsname)

    @classmethod
    def create_from_model(cls, lmf, update_mode=False):
        """Save to cache."""
        fsmodel = FileSystem(lmf)
        # xmf_path could be set later if setup_target_devices do not need it
        fsmodel.xmf_path = cls._cache_path(fsmodel.fs_name)
        fsmodel.setup_target_devices(update_mode=update_mode)
        # Save XMF
        fsmodel.model.save(fsmodel.xmf_path,
                  "# Shine Lustre file system config file for %s" % \
                  fsmodel.fs_name)

        # Reload from content saved previously
        return cls.load_from_fsname(fsmodel.fs_name)

    @classmethod
    def load_from_fsname(cls, fsname):
        """Load from cache."""
        conf_file = cls._cache_path(fsname)
        fsconf = FileSystem(conf_file)
        fsconf.xmf_path = conf_file
        return fsconf

    def _start_backend(self):
        """
        Load and start backend subsystem once
        """
        if not self.backend:

            # Start the selected config backend system.
            self.backend = BackendRegistry().selected()
            if self.backend:
                self.backend.start()

        return self.backend

    def setup_target_devices(self, update_mode=False):
        """ Generate the eXtended Model File XMF """

        self._start_backend()

        # We have to setup the possible targets, which are: MGT, MDT and OST.
        for target in ['mgt', 'mdt', 'ost']:

            # So, first, look for which ones are defined in model
            if target not in self.model:
                continue

            # Lustre supports up to FFFF targets per type.
            indexes = list(range(0, 65535))

            if self.backend:

                # Returns a list of TargetDevices
                fs_name = None
                if update_mode == True:
                    fs_name = self.fs_name
                candidates = self.backend.get_target_devices(
                    target, fs_name=fs_name, update_mode=update_mode)

                # Save the model target selection
                target_models = copy.copy(self.model.get(target))

                # Reduce entropy from backend.
                # (node, dev) should point to a unique device, so this should
                # be enough for sorting. This is not perfect but enough to have
                # a very low entropy in result order.
                candidates.sort(key=lambda x: (x.get('node'), x.get('dev')))

                # Delete it (to be replaced... see below)
                self.model.elements(target).clear()

                try:

                    # Remove already used index from candidate list.
                    for target_model in target_models:
                        if 'index' in target_model:
                            idx = target_model.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Iterates on each Model.Target
                    for target_model in target_models:

                        # Do not try to match external components
                        if target_model.get('mode') == 'external':
                            self.model.elements(target).parse(
                                str(target_model))
                            continue

                        result = target_model.match_device(candidates)
                        if len(result) == 0:
                            raise ConfigDeviceNotFoundError(target_model)

                        for matching in result:
                            candidates.remove(matching)

                            # If an index was specified, set it.
                            if 'index' in target_model:
                                matching.add_index(target_model.get('index'))
                                target_model.elements('index').clear()

                            # Manage index, mandatoy in XMF files
                            if not matching.has_index():
                                matching.add_index(indexes[0])
                                idx = matching.index()
                                # List raises ValueError and we need to know
                                # the missing indexes here, so we had this
                                # hack.
                                if idx not in indexes:
                                    raise KeyError(idx)
                                indexes.remove(idx)

                            # Copy properties from model
                            # which do not exist in backend.
                            matching.add_active(target_model.get('active'))

                            # `matching' is a TargetDevice, we want to add it
                            # to the underlying Model object. The current way
                            # to do this to create a configuration line string
                            # (performed by TargetDevice.getline()) and then
                            # call Model.parse().
                            # TODO: add methods to Model/ModelDevice to avoid
                            #  the use of temporary configuration string line.
                            self.model.elements(target).parse(
                                matching.getline())

                except KeyError as error:
                    raise ConfigInvalidFileSystem(self, \
                            "Index %s for %s used twice." % \
                            (str(error), target))

            # Support for backend None
            else:

                try:
                    # Remove already used indexes from candidate list.
                    for params in self.model.elements(target):
                        if 'index' in params:
                            idx = params.get('index')
                            # List raises ValueError and we need to know the
                            # missing indexes here, so we had this hack.
                            if idx not in indexes:
                                raise KeyError(idx)
                            indexes.remove(idx)

                    # Manage index
                    for params in self.model.elements(target):
                        if 'index' not in params:
                            params.add('index', str(indexes[0]))
                            indexes.remove(indexes[0])

                except KeyError as error:
                    raise ConfigInvalidFileSystem(self, \
                             "Index %s for %s used twice." % \
                              (str(error), target))

        self._check_coherency()

    def _check_coherency(self):
        """Verify that the declared components make a coherent filesystem."""

        model = self.model
        # If we have a target or a client, we need a MGT
        if (('client' in model or 'mdt' in model or 'ost' in model) \
            and not ('mgt' in model)):
            raise ConfigInvalidFileSystem(self, "A MGS must be declared.")

        # We should have both MDT and OST or neither
        if ('mdt' in model) ^ ('ost' in model):
            raise ConfigInvalidFileSystem(
                self, "You must declare both MDT and OST or neither.")

    def compare(self, otherfs):
        """
        Compare the FileSystem model with another FileSystem and return
        a dictionnary describing the needed actions.
        """

        actions = {}
        added, changed, removed = self.model.diff(otherfs.model)

        anyset = set(changed.keys()) | set(added.keys()) \
                  | set(removed.keys())

        # Read-only keys: fs_name
        readonly = set(['fs_name'])
        if readonly & anyset:
            raise ConfigException("%s could not be changed" %
                                  ", ".join(readonly & anyset))

        # Need to reformat targets
        reformatkeys = set([
            'mgt_mkfs_options', 'mdt_mkfs_options', 'ost_mkfs_options',
            'mgt_format_params', 'mdt_format_params', 'ost_format_params'
        ])
        if reformatkeys & anyset:
            actions['reformat'] = True

        # Need a tunefs.lustre
        tunefskeys = set([
            'quota', 'quota_type', 'quota_bunit', 'quota_iunit', 'quota_btune',
            'quota_itune', 'stripe_size', 'stripe_count'
        ])
        if tunefskeys & anyset:
            actions['tunefs'] = True

        # Need a writeconf
        writeconfkeys = set(['nid_map'])
        if writeconfkeys & anyset:
            hosts = set()
            for content in (added, changed, removed):
                hosts |= set([
                    elem.get('nodes') for elem in content.elements('nid_map')
                ])

            for tag in ('mgt', 'mdt', 'ost'):
                for tgt in self.model.get(tag, []):
                    if tgt.get('node') in hosts:
                        # Note: Target removal could also set this flag.
                        actions['writeconf'] = True
                        break

        # Need to unmount then remount clients
        remountkeys = set(['mount_options', 'mount_path', 'subdir'])
        if remountkeys & anyset:
            actions['copyconf'] = True
            # Could be improved if doing this only on clients without specific
            # path or options.
            if self.model.get('client'):
                actions['unmount'] = [
                    Clients(cli) for cli in self.model.get('client')
                ]
            if otherfs.model.get('client'):
                actions['mount'] = [
                    Clients(cli) for cli in otherfs.model.get('client')
                ]

        # Need to restart targets
        restartkeys = set([
            'mgt_mount_path', 'mdt_mount_path', 'ost_mount_path',
            'mgt_mount_options', 'mdt_mount_options', 'ost_mount_options'
        ])
        if restartkeys & anyset:
            actions['restart'] = True

        # Only need to update cache file
        copykeys = set(['description'])
        if copykeys & anyset:
            actions['copyconf'] = True

        # Clients have changed
        if 'client' in removed:
            actions.setdefault('unmount', [])
            actions['unmount'] += [
                Clients(elem) for elem in removed.elements('client')
            ]
        if 'client' in added:
            actions.setdefault('mount', [])
            actions['mount'] += [
                Clients(elem) for elem in added.elements('client')
            ]

        if 'client' in changed:
            for elem in changed.elements('client'):
                actions.setdefault('unmount', []).append(Clients(elem.old))
                actions.setdefault('mount', []).append(Clients(elem))

        # Router has changed
        if 'router' in removed:
            actions.setdefault('stop', []).extend(
                [Routers(elem) for elem in removed.elements('router')])
        if 'router' in added:
            actions.setdefault('start', []).extend(
                [Routers(elem) for elem in added.elements('router')])
        assert 'router' not in changed, 'Router change is not supported'

        # Some targets have changed
        for tgt in ['mgt', 'mdt', 'ost']:
            if tgt in removed:
                actions['writeconf'] = True
                actions.setdefault('stop', []).extend(
                    [Target(tgt, elem) for elem in removed.elements(tgt)])
                actions.setdefault('remove', []).extend(
                    [Target(tgt, elem) for elem in removed.elements(tgt)])
            if tgt in added:
                actions.setdefault('format', []).extend(
                    [Target(tgt, elem) for elem in added.elements(tgt)])
                actions.setdefault('start', []).extend(
                    [Target(tgt, elem) for elem in added.elements(tgt)])
            if tgt in changed:
                for elem in changed.elements(tgt):
                    if set(['ha_node', 'network', 'node', 'dev']) & \
                       elem.chgkeys:
                        actions.setdefault('stop',
                                           []).append(Target(tgt, elem.old))
                        actions.setdefault('start',
                                           []).append(Target(tgt, elem))

                    if set(['ha_node', 'network', 'node']) & elem.chgkeys:
                        actions['writeconf'] = True

                    if set(['tag', 'group']) & elem.chgkeys:
                        actions['copyconf'] = True

                    if 'active' in elem.chgkeys:
                        actions['tune'] = True

                    if 'jdev' in elem.chgkeys:
                        raise ConfigException("'jdev' change is not supported")

        # If some actions is required, we need to update config files.
        if len(actions) > 0:
            actions['copyconf'] = True

        return actions

    def get_nid(self, node):
        try:
            return self.nid_map[node]
        except KeyError:
            raise ConfigException("Cannot get NID for %s, aborting. Please "
                                  "verify `nid_map' configuration." % node)

    def close(self):
        if self.backend:
            self.backend.stop()
            self.backend = None

    def register(self):
        """
        This function aims to register the file system configuration
        to the backend.
        """
        if self._start_backend():
            return self.backend.register_fs(self)

    def unregister(self):
        """
        This function aims to remove a file system configuration from
        the backend.
        """
        result = 0
        if self._start_backend():
            result = self.backend.unregister_fs(self)

        # XMF file could have been already deleted if this is a local
        # filesystem
        if os.path.exists(self.xmf_path) and not result:
            os.unlink(self.xmf_path)

        return result

    def register_target(self, target):
        """
        Set the specified target as 'in use'.

        This target could not be use anymore for other filesystems.
        """
        if self._start_backend():
            return self.backend.register_target(self, target)

    def unregister_target(self, target):
        """
        Set the specified target as available in the backend.

        This target could be now reuse.
        """
        if self._start_backend():
            return self.backend.unregister_target(self, target)
Esempio n. 32
0
 def testTooLongFSName(self):
     """Model with a too long fsname"""
     testfile = makeTempFile("""fs_name: too_long_name""")
     model = Model()
     self.assertRaises(ModelFileValueError, model.load, testfile.name)
Esempio n. 33
0
 def testDefaultValues(self):
     """test defaults values"""
     m = Model()
     self.assertEqual(m.get('stripe_size'), 1048576)
     self.assertEqual(m.get('stripe_count'), 1)
     self.assertEqual(m.get('quota_type'), 'ug')
Esempio n. 34
0
 def test_multi_pattern_nid_map(self):
     """multiple patterns in nid_map is detected"""
     model = Model()
     txt = textwrap.dedent("""fs_name: nids
         nid_map: nodes=foo[1-2],bar[1-2] nids=foo[1-2]@tcp,bar[1-2]@tcp""")
     self.assertRaises(ModelFileValueError, model.parse, txt)
Esempio n. 35
0
 def test_unbalanced_nid_map(self):
     """size mismatch in nid_map raises an exception"""
     model = Model()
     txt = textwrap.dedent("""fs_name: nids
         nid_map: nodes=foo[1-2] nids=foo[1-9]@tcp""")
     self.assertRaises(ModelFileValueError, model.parse, txt)