def testUserDefinedInstanceTypes(self):
     file = StringIO.StringIO(self.VALID_CONFIG_WITH_INSTANCE_TYPES_SECTION)
     usercfg = UserConfigurator(file)
     types = usercfg.getUserDefinedInstanceTypes()
     self.assertEqual(len(types.keys()), 2)
     self.assertTrue('alpha' in types.keys())
     self.assertTrue('beta' in types.keys())
    def testWithSection(self):
        file = StringIO.StringIO(self.VALID_CONFIG_WITH_SECTION)
        configurator = UserConfigurator(file)
        dict = configurator.getDict()
        self.assertEqual(dict['username'], '<username>')

        dict = configurator.getDict("my-section")        
        self.assertEqual(dict['username'], '<another.username>')
 def testSectionDictWithDefaultSection(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     usercfg = UserConfigurator(file)
     values = usercfg.getSectionDict('default')
     self.assertEqual(len(values), 3)
     self.assertEqual(values['endpoint'], '<cloud.frontend.hostname>')
     self.assertEqual(values['username'], '<username>')
     self.assertEqual(values['password'], '<password>')
Example #4
0
    def __init__(self, key, secret=None, secure=False, host=None, port=None,
                 api_version=None, **kwargs):
        """
        Creates a new instance of a StratusLabNodeDriver from the
        given parameters.  All of the parameters are ignored except
        for the ones defined below.

        The configuration is read from the named configuration file
        (or file-like object).  The 'locations' in the API correspond
        to the named sections within the configuration file.

        :param key: ignored by this driver
        :param secret: ignored by this driver
        :param secure: passed to superclass; True required CA certs
        :param host: ignored by this driver (use locations instead)
        :param port: ignored by this driver (use locations instead)
        :param api_version: ignored by this driver
        :param kwargs: additional keyword arguments

        :keyword stratuslab_user_config (str or file): File name or
        file-like object from which to read the user's StratusLab
        configuration.  Sections in the configuration file correspond
        to 'locations' within this API.

        :keyword stratuslab_default_location (str): The id (name) of
        the section within the user configuration file to use as the
        default location.

        :returns: StratusLabNodeDriver

        """
        super(StratusLabNodeDriver, self).__init__(key,
                                                   secret=secret,
                                                   secure=secure,
                                                   host=host,
                                                   port=port,
                                                   api_version=api_version,
                                                   **kwargs)

        self.name = "StratusLab Node Provider"
        self.website = 'http://stratuslab.eu/'
        self.type = "stratuslab"

        # only ssh-based authentication is supported by StratusLab
        self.features['create_node'] = ['ssh_key']

        user_config_file = kwargs.get('stratuslab_user_config',
                                      StratusLabUtil.defaultConfigFileUser)
        default_section = kwargs.get('stratuslab_default_location', None)

        self.user_configurator = UserConfigurator(configFile=user_config_file)

        self.default_location, self.locations = \
            self._get_config_locations(default_section)

        self.sizes = self._get_config_sizes()
 def testStaticLoaderWithNamedFile(self):
     file = self._createTemporaryFile(self.VALID_CONFIG_DEFAULT_ONLY)
     try:
         configHolder = UserConfigurator.configFileToDictWithFormattedKeys(file)
     finally:
         os.remove(file)
     self.assertEqual(configHolder['username'], '<username>')
 def testValidTuples(self):
     testValues = [ (1,2,3),
                    (1,1,0),
                    (24,256,1024)
                    ]
     for t in testValues:
         result = UserConfigurator._validInstanceTypeTuple(t);
         self.assertTrue(result, 'valid tuple marked as invalid: %s' % str(t))
Example #7
0
    def __init__(self, key, secret=None, secure=False, host=None, port=None,
                 api_version=None, **kwargs):
        """
        Creates a new instance of a StratusLabNodeDriver from the
        given parameters.  All of the parameters are ignored except
        for the ones defined below.

        The configuration is read from the named configuration file
        (or file-like object).  The 'locations' in the API correspond
        to the named sections within the configuration file.

        :param key: ignored by this driver
        :param secret: ignored by this driver
        :param secure: passed to superclass; True required CA certs
        :param host: ignored by this driver (use locations instead)
        :param port: ignored by this driver (use locations instead)
        :param api_version: ignored by this driver
        :param kwargs: additional keyword arguments

        :keyword stratuslab_user_config (str or file): File name or
        file-like object from which to read the user's StratusLab
        configuration.  Sections in the configuration file correspond
        to 'locations' within this API.

        :keyword stratuslab_default_location (str): The id (name) of
        the section within the user configuration file to use as the
        default location.

        :returns: StratusLabNodeDriver

        """
        super(StratusLabNodeDriver, self).__init__(key,
                                                   secret=secret,
                                                   secure=secure,
                                                   host=host,
                                                   port=port,
                                                   api_version=api_version,
                                                   **kwargs)

        self.name = "StratusLab Node Provider"
        self.website = 'http://stratuslab.eu/'
        self.type = "stratuslab"

        # only ssh-based authentication is supported by StratusLab
        self.features['create_node'] = ['ssh_key']

        user_config_file = kwargs.get('stratuslab_user_config',
                                      StratusLabUtil.defaultConfigFileUser)
        default_section = kwargs.get('stratuslab_default_location', None)

        self.user_configurator = UserConfigurator(configFile=user_config_file)

        self.default_location, self.locations = \
            self._get_config_locations(default_section)

        self.sizes = self._get_config_sizes()
 def testStringToTupleConversion(self):
     testValues = { '1,2,3': (1,2,3),
                    '1,,2,,3': (1,2,3),
                    '': (),
                    ',c,c,c': (),
                    '(4,4,4)': (4,4,4),
                    '(0,0,0,0,)': (0,0,0,0),
                    }
     for s in testValues.keys():
         trueValue = testValues[s]
         t = UserConfigurator._instanceTypeStringToTuple(s);
         self.assertEqual(t, trueValue, 'incorrect value mapping: %s, %s, %s' % (s, t, trueValue))
Example #9
0
    def get_config_section(location, user_configurator, options=None):

        config = UserConfigurator.userConfiguratorToDictWithFormattedKeys(
            user_configurator, selected_section=location.id)

        options = options or {}
        options['verboseLevel'] = -1
        options['verbose_level'] = -1

        configHolder = ConfigHolder(options=(options or {}), config=config)
        configHolder.pdiskProtocol = 'https'

        return configHolder
Example #10
0
    def get_config_section(location, user_configurator, options=None):

        config = UserConfigurator.userConfiguratorToDictWithFormattedKeys(user_configurator,
                                                                          selected_section=location.id)

        options = options or {}
        options['verboseLevel'] = -1
        options['verbose_level'] = -1

        configHolder = ConfigHolder(options=(options or {}), config=config)
        configHolder.pdiskProtocol = 'https'

        return configHolder
Example #11
0
 def testInvalidTuples(self):
     testValues = [ (0,1,0),
                    (1,0,0),
                    (),
                    (1,2,3,4),
                    (-1,1,0),
                    (1,-1,0),
                    (1,1,-1),
                    ('a',2,3),
                    (1,'a',3),
                    (1,2,'a')
                    ]
     for t in testValues:
         result = UserConfigurator._validInstanceTypeTuple(t);
         self.assertFalse(result, 'invalid tuple marked as valid: %s' % str(t))
Example #12
0
class StratusLabNodeDriver(NodeDriver):
    """StratusLab node driver."""

    RDF_RDF = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF'
    RDF_DESCRIPTION = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'
    DC_IDENTIFIER = '{http://purl.org/dc/terms/}identifier'
    DC_TITLE = '{http://purl.org/dc/terms/}title'
    DC_DESCRIPTION = '{http://purl.org/dc/terms/}description'

    DEFAULT_MARKETPLACE_URL = 'https://marketplace.stratuslab.eu'

    user_configurator = None
    locations = None
    default_location = None
    sizes = None

    def __init__(self, key, secret=None, secure=False, host=None, port=None,
                 api_version=None, **kwargs):
        """
        Creates a new instance of a StratusLabNodeDriver from the
        given parameters.  All of the parameters are ignored except
        for the ones defined below.

        The configuration is read from the named configuration file
        (or file-like object).  The 'locations' in the API correspond
        to the named sections within the configuration file.

        :param key: ignored by this driver
        :param secret: ignored by this driver
        :param secure: passed to superclass; True required CA certs
        :param host: ignored by this driver (use locations instead)
        :param port: ignored by this driver (use locations instead)
        :param api_version: ignored by this driver
        :param kwargs: additional keyword arguments

        :keyword stratuslab_user_config (str or file): File name or
        file-like object from which to read the user's StratusLab
        configuration.  Sections in the configuration file correspond
        to 'locations' within this API.

        :keyword stratuslab_default_location (str): The id (name) of
        the section within the user configuration file to use as the
        default location.

        :returns: StratusLabNodeDriver

        """
        super(StratusLabNodeDriver, self).__init__(key,
                                                   secret=secret,
                                                   secure=secure,
                                                   host=host,
                                                   port=port,
                                                   api_version=api_version,
                                                   **kwargs)

        self.name = "StratusLab Node Provider"
        self.website = 'http://stratuslab.eu/'
        self.type = "stratuslab"

        # only ssh-based authentication is supported by StratusLab
        self.features['create_node'] = ['ssh_key']

        user_config_file = kwargs.get('stratuslab_user_config',
                                      StratusLabUtil.defaultConfigFileUser)
        default_section = kwargs.get('stratuslab_default_location', None)

        self.user_configurator = UserConfigurator(configFile=user_config_file)

        self.default_location, self.locations = \
            self._get_config_locations(default_section)

        self.sizes = self._get_config_sizes()

    # noinspection PyUnusedLocal
    def get_uuid(self, unique_field=None):
        """
        :param  unique_field (bool): Unique field
        :returns: UUID

        """
        return str(uuid.uuid4())

    @staticmethod
    def get_config_section(location, user_configurator, options=None):

        config = UserConfigurator.userConfiguratorToDictWithFormattedKeys(user_configurator,
                                                                          selected_section=location.id)

        options = options or {}
        options['verboseLevel'] = -1
        options['verbose_level'] = -1

        configHolder = ConfigHolder(options=(options or {}), config=config)
        configHolder.pdiskProtocol = 'https'

        return configHolder

    def _get_config_section(self, location, options=None):
        location = location or self.default_location
        return StratusLabNodeDriver.get_config_section(location,
                                                       self.user_configurator,
                                                       options)

    def _get_config_locations(self, default_section=None):
        """
        Returns the default location and a dictionary of locations.

        Locations are defined as sections in the client configuration
        file.  The sections may contain 'name' and 'country' keys.  If
        'name' is not present, then the id is also used for the name.
        If 'country' is not present, then 'unknown' is the default
        value.

        The default location in order of preference is: named section
        in parameter, selected_section in configuration, and finally
        'default'.  The 'default' key will remain the returned
        dictionary only if it is the chosen default location.

        """

        # TODO: Decide to make parser public or provide method for this info.
        parser = self.user_configurator._parser

        # determine the default location (section) to use
        # preference: parameter, selected section in config., [default] section
        if not default_section:
            try:
                default_section = parser.get('default', 'selected_section')
            except ConfigParser.NoOptionError:
                default_section = 'default'
            except ConfigParser.NoSectionError:
                raise Exception('configuration file must have [default] section')

        locations = {}

        for section in parser.sections():
            if not (section in ['instance_types']):
                location_id = section

                try:
                    name = parser.get(section, 'name')
                except ConfigParser.NoOptionError:
                    name = location_id

                try:
                    country = parser.get(section, 'country')
                except ConfigParser.NoOptionError:
                    country = 'unknown'

                locations[location_id] = NodeLocation(id=section,
                                                      name=name,
                                                      country=country,
                                                      driver=self)

        try:
            default_location = locations[default_section]
        except KeyError:
            raise Exception('requested default location (%s) not defined' %
                            default_section)

        if default_section != 'default':
            del(locations['default'])

        return default_location, locations

    def _get_config_sizes(self):
        """
        Create all of the node sizes based on the user configuration.

        """

        size_map = {}

        machine_types = Runner.getDefaultInstanceTypes()
        for name in machine_types.keys():
            size = self._create_node_size(name, machine_types[name])
            size_map[name] = size

        machine_types = self.user_configurator.getUserDefinedInstanceTypes()
        for name in machine_types.keys():
            size = self._create_node_size(name, machine_types[name])
            size_map[name] = size

        return size_map.values()

    def _create_node_size(self, name, resources):
        cpu, ram, swap = resources
        bandwidth = 1000
        price = 1
        return StratusLabNodeSize(size_id=name,
                                  name=name,
                                  ram=ram,
                                  disk=swap,
                                  bandwidth=bandwidth,
                                  price=price,
                                  driver=self,
                                  cpu=cpu)

    def list_nodes(self):
        """
        List the nodes (machine instances) that are active in all
        locations.

        """

        nodes = []
        for location in self.locations.values():
            nodes.extend(self.list_nodes_in_location(location))
        return nodes

    def list_nodes_in_location(self, location):
        """
        List the nodes (machine instances) that are active in the
        given location.

        """

        configHolder = self._get_config_section(location)

        monitor = Monitor(configHolder)
        vms = monitor.listVms()

        nodes = []
        for vm_info in vms:
            nodes.append(self._vm_info_to_node(vm_info, location))

        return nodes

    def _vm_info_to_node(self, vm_info, location):
        attrs = vm_info.getAttributes()
        node_id = attrs['id'] or None
        name = attrs['name'] or None
        state = StratusLabNodeDriver._to_node_state(attrs['state_summary'] or None)

        public_ip = attrs['template_nic_ip']
        if public_ip:
            public_ips = [public_ip]
        else:
            public_ips = []

        size_name = '%s_size' % node_id

        cpu = attrs['template_cpu']
        ram = attrs['template_memory']
        swap = attrs['template_disk_size']

        size = self._create_node_size(size_name, (cpu, ram, swap))

        mp_url = attrs['template_disk_source']
        mp_id = mp_url.split('/')[-1]
        image = NodeImage(mp_id, mp_id, self)

        return StratusLabNode(node_id,
                              name,
                              state,
                              public_ips,
                              None,
                              self,
                              size=size,
                              image=image,
                              extra={'location': location})

    @staticmethod
    def _to_node_state(state):
        if state:
            state = state.lower()
            if state in ['running', 'epilog']:
                return NodeState.RUNNING
            elif state in ['pending', 'prolog', 'boot']:
                return NodeState.PENDING
            elif state in ['done']:
                return NodeState.TERMINATED
            else:
                return NodeState.UNKNOWN
        else:
            return NodeState.UNKNOWN

    def create_node(self, **kwargs):
        """
        @keyword    name:   String with a name for this new node (required)
        @type       name:   C{str}

        @keyword    size:   The size of resources allocated to this node.
                            (required)
        @type       size:   L{NodeSize}

        @keyword    image:  OS Image to boot on node. (required)
        @type       image:  L{NodeImage}

        @keyword    location: Which data center to create a node in. If empty,
                              undefined behavoir will be selected. (optional)
        @type       location: L{NodeLocation}

        @keyword    auth:   Initial authentication information for the node
                            (optional)
        @type       auth:   L{NodeAuthSSHKey} or L{NodeAuthPassword}

        @return: The newly created node.
        @rtype: L{Node}

        @inherits: L{NodeDriver.create_node}

        """

        name = kwargs.get('name')
        size = kwargs.get('size')
        image = kwargs.get('image')
        location = kwargs.get('location', self.default_location)
        auth = kwargs.get('auth', None)

        runner = self._create_runner(name, size, image,
                                     location=location, auth=auth)

        ids = runner.runInstance()
        node_id = ids[0]

        extra = {'location': location}

        node = StratusLabNode(node_id=node_id,
                              name=name,
                              state=NodeState.PENDING,
                              public_ips=[],
                              private_ips=[],
                              driver=self,
                              size=size,
                              image=image,
                              extra=extra)

        try:
            _, ip = runner.getNetworkDetail(node_id)
            node.public_ips = [ip]

        except Exception as e:
            print e

        return node

    def _create_runner(self, name, size, image, location=None, auth=None):

        location = location or self.default_location

        holder = self._get_config_section(location)

        self._insert_required_run_option_defaults(holder)

        holder.set('vmName', name)

        pubkey_file = None
        if isinstance(auth, NodeAuthSSHKey):
            _, pubkey_file = tempfile.mkstemp(suffix='_pub.key', prefix='ssh_')
            with open(pubkey_file, 'w') as f:
                f.write(auth.pubkey)

            holder.set('userPublicKeyFile', pubkey_file)

        # The cpu attribute is only included in the StratusLab
        # subclass of NodeSize.  Recover if the user passed in a
        # normal NodeSize; default to 1 CPU in this case.
        try:
            cpu = size.cpu
        except AttributeError:
            cpu = 1

        holder.set('vmCpu', cpu)
        holder.set('vmRam', size.ram)
        holder.set('vmSwap', size.disk)

        runner = Runner(image.id, holder)

        if pubkey_file:
            os.remove(pubkey_file)

        return runner

    def _insert_required_run_option_defaults(self, holder):
        defaults = Runner.defaultRunOptions()

        defaults['verboseLevel'] = -1
        required_options = ['verboseLevel', 'vmTemplateFile',
                            'marketplaceEndpoint', 'vmRequirements',
                            'outVmIdsFile', 'inVmIdsFile']

        for option in required_options:
            if not holder.config.get(option):
                holder.config[option] = defaults[option]

    def destroy_node(self, node):
        """
        Terminate the node and remove it from the node list.  This is
        the equivalent of stratus-kill-instance.

        """

        runner = self._create_runner(node.name, node.size, node.image,
                                     location=node.location)
        runner.killInstances([node.id])

        node.state = NodeState.TERMINATED

        return True

    def list_images(self, location=None):
        """
        Returns a list of images from the StratusLab Marketplace.  The
        image id corresponds to the base64 identifier of the image in
        the Marketplace and the name corresponds to the title (or
        description if title isn't present).

        The location parameter is ignored at the moment and the global
        Marketplace (https://marketplace.stratuslab.eu/metadata) is
        consulted.

        @inherits: L{NodeDriver.list_images}
        """

        location = location or self.default_location

        holder = self._get_config_section(location)
        url = holder.config.get('marketplaceEndpoint',
                                self.DEFAULT_MARKETPLACE_URL)
        endpoint = '%s/metadata' % url
        return self._get_marketplace_images(endpoint)

    def _get_marketplace_images(self, url):
        images = []
        try:
            filename, _ = urllib.urlretrieve(url)
            tree = ET.parse(filename)
            root = tree.getroot()
            for md in root.findall(self.RDF_RDF):
                rdf_desc = md.find(self.RDF_DESCRIPTION)
                image_id = rdf_desc.find(self.DC_IDENTIFIER).text
                elem = rdf_desc.find(self.DC_TITLE)
                if elem is None or len(elem) == 0:
                    elem = rdf_desc.find(self.DC_DESCRIPTION)

                if elem is not None and elem.text is not None:
                    name = elem.text.lstrip()[:30]
                else:
                    name = ''
                images.append(NodeImage(id=image_id, name=name, driver=self))
        except Exception as e:
            # TODO: log errors instead of ignoring them
            print e

        return images

    def list_sizes(self, location=None):
        """
        StratusLab node sizes are defined by the client and do not
        depend on the location.  Consequently, the location parameter
        is ignored.  Node sizes defined in the configuration file
        (in the 'instance_types' section) augment or replace the
        standard node sizes defined by default.

        @inherits: L{NodeDriver.list_images}
        """
        return self.sizes

    def list_locations(self):
        """
        Returns a list of StratusLab locations.  These are defined as
        sections in the client configuration file.  The sections may
        contain 'name' and 'country' keys.  If 'name' is not present,
        then the id is also used for the name.  If 'country' is not
        present, then 'unknown' is the default value.

        The returned list and contained NodeLocations are not
        intended to be modified by the user.

        @inherits: L{NodeDriver.list_locations}
        """
        return self.locations.values()

    def list_volumes(self, location=None):
        """
        Creates a list of all of the volumes in the given location.
        This will include private disks of the user as well as public
        disks from other users.

        This method is not a standard part of the Libcloud node driver
        interface.
        """

        configHolder = self._get_config_section(location)

        pdisk = PersistentDisk(configHolder)

        filters = {}
        volumes = pdisk.describeVolumes(filters)

        storage_volumes = []
        for info in volumes:
            storage_volumes.append(self._create_storage_volume(info, location))

        return storage_volumes

    def _create_storage_volume(self, info, location):
        disk_uuid = info['uuid']
        name = info['tag']
        size = info['size']
        extra = {'location': location}
        return StorageVolume(disk_uuid, name, size, self, extra=extra)

    def create_volume(self, size, name, location=None, snapshot=None):
        """
        Creates a new storage volume with the given size.  The 'name'
        corresponds to the volume tag.  The visibility of the created
        volume is 'private'.

        The snapshot parameter is currently ignored.

        The created StorageVolume contains a dict for the extra
        information with a 'location' key storing the location used
        for the volume.  This is set to 'default' if no location has
        been given.

        @inherits: L{NodeDriver.create_volume}
        """
        configHolder = self._get_config_section(location)

        pdisk = PersistentDisk(configHolder)

        # Creates a private disk.  Boolean flag = False means private.
        vol_uuid = pdisk.createVolume(size, name, False)

        extra = {'location': location}

        return StorageVolume(vol_uuid, name, size, self, extra=extra)

    def destroy_volume(self, volume):
        """
        Destroys the given volume.

        @inherits: L{NodeDriver.destroy_volume}
        """

        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        pdisk.deleteVolume(volume.id)

        return True

    def attach_volume(self, node, volume, device=None):
        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        try:
            host = node.host
        except AttributeError:
            raise Exception('node does not contain host information')

        pdisk.hotAttach(host, node.id, volume.id)

        try:
            volume.extra['node'] = node
        except AttributeError:
            volume.extra = {'node': node}

        return True

    def detach_volume(self, volume):

        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        try:
            node = volume.extra['node']
        except (AttributeError, KeyError):
            raise Exception('volume is not attached to a node')

        pdisk.hotDetach(node.id, volume.id)

        del(volume.extra['node'])

        return True

    def _volume_location(self, volume):
        """
        Recovers the location information from the volume.  If
        the information is not available, then the default
        location for this driver is used.

        """

        try:
            return volume.extra['location']
        except KeyError:
            return self.default_location
Example #13
0
 def testStaticLoader(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     configHolder = UserConfigurator.configFileToDictWithFormattedKeys(file)
     self.assertEqual(configHolder['username'], '<username>')
Example #14
0
 def testDefaultInstanceTypeWithoutConfigAttribute(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     usercfg = UserConfigurator(file)
     dict = usercfg.getDict()
     self.assertFalse('default_instance_type' in dict.keys())
Example #15
0
 def testWithSectionAndReference(self):
     file = StringIO.StringIO(self.VALID_CONFIG_WITH_SECTION_AND_REFERENCE)
     configurator = UserConfigurator(file)
     dict = configurator.getDict()
     self.assertEqual(dict['username'], '<another.username>')
Example #16
0
 def testDefaultOnly(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     configurator = UserConfigurator(file)
     dict = configurator.getDict()
     self.assertEqual(dict['username'], '<username>')
Example #17
0
 def testDefaultInstanceTypeWithConfigAttribute(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY_WITH_INSTANCE_TYPE)
     usercfg = UserConfigurator(file)
     dict = usercfg.getDict()
     self.assertEqual(dict['default_instance_type'], 'my.type')
Example #18
0
 def testSectionDictWithNonexistentSection(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     usercfg = UserConfigurator(file)
     values = usercfg.getSectionDict('non-existent-section')
     self.assertEqual(len(values), 0)
Example #19
0
 def testSectionDictWithNoArgument(self):
     file = StringIO.StringIO(self.VALID_CONFIG_DEFAULT_ONLY)
     usercfg = UserConfigurator(file)
     values = usercfg.getSectionDict()
     self.assertEqual(len(values), 0)
Example #20
0
class StratusLabNodeDriver(NodeDriver):
    """StratusLab node driver."""

    RDF_RDF = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF'
    RDF_DESCRIPTION = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'
    DC_IDENTIFIER = '{http://purl.org/dc/terms/}identifier'
    DC_TITLE = '{http://purl.org/dc/terms/}title'
    DC_DESCRIPTION = '{http://purl.org/dc/terms/}description'

    DEFAULT_MARKETPLACE_URL = 'https://marketplace.stratuslab.eu'

    user_configurator = None
    locations = None
    default_location = None
    sizes = None

    def __init__(self,
                 key,
                 secret=None,
                 secure=False,
                 host=None,
                 port=None,
                 api_version=None,
                 **kwargs):
        """
        Creates a new instance of a StratusLabNodeDriver from the
        given parameters.  All of the parameters are ignored except
        for the ones defined below.

        The configuration is read from the named configuration file
        (or file-like object).  The 'locations' in the API correspond
        to the named sections within the configuration file.

        :param key: ignored by this driver
        :param secret: ignored by this driver
        :param secure: passed to superclass; True required CA certs
        :param host: ignored by this driver (use locations instead)
        :param port: ignored by this driver (use locations instead)
        :param api_version: ignored by this driver
        :param kwargs: additional keyword arguments

        :keyword stratuslab_user_config (str or file): File name or
        file-like object from which to read the user's StratusLab
        configuration.  Sections in the configuration file correspond
        to 'locations' within this API.

        :keyword stratuslab_default_location (str): The id (name) of
        the section within the user configuration file to use as the
        default location.

        :returns: StratusLabNodeDriver

        """
        super(StratusLabNodeDriver, self).__init__(key,
                                                   secret=secret,
                                                   secure=secure,
                                                   host=host,
                                                   port=port,
                                                   api_version=api_version,
                                                   **kwargs)

        self.name = "StratusLab Node Provider"
        self.website = 'http://stratuslab.eu/'
        self.type = "stratuslab"

        # only ssh-based authentication is supported by StratusLab
        self.features['create_node'] = ['ssh_key']

        user_config_file = kwargs.get('stratuslab_user_config',
                                      StratusLabUtil.defaultConfigFileUser)
        default_section = kwargs.get('stratuslab_default_location', None)

        self.user_configurator = UserConfigurator(configFile=user_config_file)

        self.default_location, self.locations = \
            self._get_config_locations(default_section)

        self.sizes = self._get_config_sizes()

    # noinspection PyUnusedLocal
    def get_uuid(self, unique_field=None):
        """
        :param  unique_field (bool): Unique field
        :returns: UUID

        """
        return str(uuid.uuid4())

    @staticmethod
    def get_config_section(location, user_configurator, options=None):

        config = UserConfigurator.userConfiguratorToDictWithFormattedKeys(
            user_configurator, selected_section=location.id)

        options = options or {}
        options['verboseLevel'] = -1
        options['verbose_level'] = -1

        configHolder = ConfigHolder(options=(options or {}), config=config)
        configHolder.pdiskProtocol = 'https'

        return configHolder

    def _get_config_section(self, location, options=None):
        location = location or self.default_location
        return StratusLabNodeDriver.get_config_section(location,
                                                       self.user_configurator,
                                                       options)

    def _get_config_locations(self, default_section=None):
        """
        Returns the default location and a dictionary of locations.

        Locations are defined as sections in the client configuration
        file.  The sections may contain 'name' and 'country' keys.  If
        'name' is not present, then the id is also used for the name.
        If 'country' is not present, then 'unknown' is the default
        value.

        The default location in order of preference is: named section
        in parameter, selected_section in configuration, and finally
        'default'.  The 'default' key will remain the returned
        dictionary only if it is the chosen default location.

        """

        # TODO: Decide to make parser public or provide method for this info.
        parser = self.user_configurator._parser

        # determine the default location (section) to use
        # preference: parameter, selected section in config., [default] section
        if not default_section:
            try:
                default_section = parser.get('default', 'selected_section')
            except ConfigParser.NoOptionError:
                default_section = 'default'
            except ConfigParser.NoSectionError:
                raise Exception(
                    'configuration file must have [default] section')

        locations = {}

        for section in parser.sections():
            if not (section in ['instance_types']):
                location_id = section

                try:
                    name = parser.get(section, 'name')
                except ConfigParser.NoOptionError:
                    name = location_id

                try:
                    country = parser.get(section, 'country')
                except ConfigParser.NoOptionError:
                    country = 'unknown'

                locations[location_id] = NodeLocation(id=section,
                                                      name=name,
                                                      country=country,
                                                      driver=self)

        try:
            default_location = locations[default_section]
        except KeyError:
            raise Exception('requested default location (%s) not defined' %
                            default_section)

        if default_section != 'default':
            del (locations['default'])

        return default_location, locations

    def _get_config_sizes(self):
        """
        Create all of the node sizes based on the user configuration.

        """

        size_map = {}

        machine_types = Runner.getDefaultInstanceTypes()
        for name in machine_types.keys():
            size = self._create_node_size(name, machine_types[name])
            size_map[name] = size

        machine_types = self.user_configurator.getUserDefinedInstanceTypes()
        for name in machine_types.keys():
            size = self._create_node_size(name, machine_types[name])
            size_map[name] = size

        return size_map.values()

    def _create_node_size(self, name, resources):
        cpu, ram, swap = resources
        bandwidth = 1000
        price = 1
        return StratusLabNodeSize(size_id=name,
                                  name=name,
                                  ram=ram,
                                  disk=swap,
                                  bandwidth=bandwidth,
                                  price=price,
                                  driver=self,
                                  cpu=cpu)

    def list_nodes(self):
        """
        List the nodes (machine instances) that are active in all
        locations.

        """

        nodes = []
        for location in self.locations.values():
            nodes.extend(self.list_nodes_in_location(location))
        return nodes

    def list_nodes_in_location(self, location):
        """
        List the nodes (machine instances) that are active in the
        given location.

        """

        configHolder = self._get_config_section(location)

        monitor = Monitor(configHolder)
        vms = monitor.listVms()

        nodes = []
        for vm_info in vms:
            nodes.append(self._vm_info_to_node(vm_info, location))

        return nodes

    def _vm_info_to_node(self, vm_info, location):
        attrs = vm_info.getAttributes()
        node_id = attrs['id'] or None
        name = attrs['name'] or None
        state = StratusLabNodeDriver._to_node_state(attrs['state_summary']
                                                    or None)

        public_ip = attrs['template_nic_ip']
        if public_ip:
            public_ips = [public_ip]
        else:
            public_ips = []

        size_name = '%s_size' % node_id

        cpu = attrs['template_cpu']
        ram = attrs['template_memory']
        swap = attrs['template_disk_size']

        size = self._create_node_size(size_name, (cpu, ram, swap))

        mp_url = attrs['template_disk_source']
        mp_id = mp_url.split('/')[-1]
        image = NodeImage(mp_id, mp_id, self)

        return StratusLabNode(node_id,
                              name,
                              state,
                              public_ips,
                              None,
                              self,
                              size=size,
                              image=image,
                              extra={'location': location})

    @staticmethod
    def _to_node_state(state):
        if state:
            state = state.lower()
            if state in ['running', 'epilog']:
                return NodeState.RUNNING
            elif state in ['pending', 'prolog', 'boot']:
                return NodeState.PENDING
            elif state in ['done']:
                return NodeState.TERMINATED
            else:
                return NodeState.UNKNOWN
        else:
            return NodeState.UNKNOWN

    def create_node(self, **kwargs):
        """
        @keyword    name:   String with a name for this new node (required)
        @type       name:   C{str}

        @keyword    size:   The size of resources allocated to this node.
                            (required)
        @type       size:   L{NodeSize}

        @keyword    image:  OS Image to boot on node. (required)
        @type       image:  L{NodeImage}

        @keyword    location: Which data center to create a node in. If empty,
                              undefined behavoir will be selected. (optional)
        @type       location: L{NodeLocation}

        @keyword    auth:   Initial authentication information for the node
                            (optional)
        @type       auth:   L{NodeAuthSSHKey} or L{NodeAuthPassword}

        @return: The newly created node.
        @rtype: L{Node}

        @inherits: L{NodeDriver.create_node}

        """

        name = kwargs.get('name')
        size = kwargs.get('size')
        image = kwargs.get('image')
        location = kwargs.get('location', self.default_location)
        auth = kwargs.get('auth', None)

        runner = self._create_runner(name,
                                     size,
                                     image,
                                     location=location,
                                     auth=auth)

        ids = runner.runInstance()
        node_id = ids[0]

        extra = {'location': location}

        node = StratusLabNode(node_id=node_id,
                              name=name,
                              state=NodeState.PENDING,
                              public_ips=[],
                              private_ips=[],
                              driver=self,
                              size=size,
                              image=image,
                              extra=extra)

        try:
            _, ip = runner.getNetworkDetail(node_id)
            node.public_ips = [ip]

        except Exception as e:
            print e

        return node

    def _create_runner(self, name, size, image, location=None, auth=None):

        location = location or self.default_location

        holder = self._get_config_section(location)

        self._insert_required_run_option_defaults(holder)

        holder.set('vmName', name)

        pubkey_file = None
        if isinstance(auth, NodeAuthSSHKey):
            _, pubkey_file = tempfile.mkstemp(suffix='_pub.key', prefix='ssh_')
            with open(pubkey_file, 'w') as f:
                f.write(auth.pubkey)

            holder.set('userPublicKeyFile', pubkey_file)

        # The cpu attribute is only included in the StratusLab
        # subclass of NodeSize.  Recover if the user passed in a
        # normal NodeSize; default to 1 CPU in this case.
        try:
            cpu = size.cpu
        except AttributeError:
            cpu = 1

        holder.set('vmCpu', cpu)
        holder.set('vmRam', size.ram)
        holder.set('vmSwap', size.disk)

        runner = Runner(image.id, holder)

        if pubkey_file:
            os.remove(pubkey_file)

        return runner

    def _insert_required_run_option_defaults(self, holder):
        defaults = Runner.defaultRunOptions()

        defaults['verboseLevel'] = -1
        required_options = [
            'verboseLevel', 'vmTemplateFile', 'marketplaceEndpoint',
            'vmRequirements', 'outVmIdsFile', 'inVmIdsFile'
        ]

        for option in required_options:
            if not holder.config.get(option):
                holder.config[option] = defaults[option]

    def destroy_node(self, node):
        """
        Terminate the node and remove it from the node list.  This is
        the equivalent of stratus-kill-instance.

        """

        runner = self._create_runner(node.name,
                                     node.size,
                                     node.image,
                                     location=node.location)
        runner.killInstances([node.id])

        node.state = NodeState.TERMINATED

        return True

    def list_images(self, location=None):
        """
        Returns a list of images from the StratusLab Marketplace.  The
        image id corresponds to the base64 identifier of the image in
        the Marketplace and the name corresponds to the title (or
        description if title isn't present).

        The location parameter is ignored at the moment and the global
        Marketplace (https://marketplace.stratuslab.eu/metadata) is
        consulted.

        @inherits: L{NodeDriver.list_images}
        """

        location = location or self.default_location

        holder = self._get_config_section(location)
        url = holder.config.get('marketplaceEndpoint',
                                self.DEFAULT_MARKETPLACE_URL)
        endpoint = '%s/metadata' % url
        return self._get_marketplace_images(endpoint)

    def _get_marketplace_images(self, url):
        images = []
        try:
            filename, _ = urllib.urlretrieve(url)
            tree = ET.parse(filename)
            root = tree.getroot()
            for md in root.findall(self.RDF_RDF):
                rdf_desc = md.find(self.RDF_DESCRIPTION)
                image_id = rdf_desc.find(self.DC_IDENTIFIER).text
                elem = rdf_desc.find(self.DC_TITLE)
                if elem is None or len(elem) == 0:
                    elem = rdf_desc.find(self.DC_DESCRIPTION)

                if elem is not None and elem.text is not None:
                    name = elem.text.lstrip()[:30]
                else:
                    name = ''
                images.append(NodeImage(id=image_id, name=name, driver=self))
        except Exception as e:
            # TODO: log errors instead of ignoring them
            print e

        return images

    def list_sizes(self, location=None):
        """
        StratusLab node sizes are defined by the client and do not
        depend on the location.  Consequently, the location parameter
        is ignored.  Node sizes defined in the configuration file
        (in the 'instance_types' section) augment or replace the
        standard node sizes defined by default.

        @inherits: L{NodeDriver.list_images}
        """
        return self.sizes

    def list_locations(self):
        """
        Returns a list of StratusLab locations.  These are defined as
        sections in the client configuration file.  The sections may
        contain 'name' and 'country' keys.  If 'name' is not present,
        then the id is also used for the name.  If 'country' is not
        present, then 'unknown' is the default value.

        The returned list and contained NodeLocations are not
        intended to be modified by the user.

        @inherits: L{NodeDriver.list_locations}
        """
        return self.locations.values()

    def list_volumes(self, location=None):
        """
        Creates a list of all of the volumes in the given location.
        This will include private disks of the user as well as public
        disks from other users.

        This method is not a standard part of the Libcloud node driver
        interface.
        """

        configHolder = self._get_config_section(location)

        pdisk = PersistentDisk(configHolder)

        filters = {}
        volumes = pdisk.describeVolumes(filters)

        storage_volumes = []
        for info in volumes:
            storage_volumes.append(self._create_storage_volume(info, location))

        return storage_volumes

    def _create_storage_volume(self, info, location):
        disk_uuid = info['uuid']
        name = info['tag']
        size = info['size']
        extra = {'location': location}
        return StorageVolume(disk_uuid, name, size, self, extra=extra)

    def create_volume(self, size, name, location=None, snapshot=None):
        """
        Creates a new storage volume with the given size.  The 'name'
        corresponds to the volume tag.  The visibility of the created
        volume is 'private'.

        The snapshot parameter is currently ignored.

        The created StorageVolume contains a dict for the extra
        information with a 'location' key storing the location used
        for the volume.  This is set to 'default' if no location has
        been given.

        @inherits: L{NodeDriver.create_volume}
        """
        configHolder = self._get_config_section(location)

        pdisk = PersistentDisk(configHolder)

        # Creates a private disk.  Boolean flag = False means private.
        vol_uuid = pdisk.createVolume(size, name, False)

        extra = {'location': location}

        return StorageVolume(vol_uuid, name, size, self, extra=extra)

    def destroy_volume(self, volume):
        """
        Destroys the given volume.

        @inherits: L{NodeDriver.destroy_volume}
        """

        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        pdisk.deleteVolume(volume.id)

        return True

    def attach_volume(self, node, volume, device=None):
        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        try:
            host = node.host
        except AttributeError:
            raise Exception('node does not contain host information')

        pdisk.hotAttach(host, node.id, volume.id)

        try:
            volume.extra['node'] = node
        except AttributeError:
            volume.extra = {'node': node}

        return True

    def detach_volume(self, volume):

        location = self._volume_location(volume)

        configHolder = self._get_config_section(location)
        pdisk = PersistentDisk(configHolder)

        try:
            node = volume.extra['node']
        except (AttributeError, KeyError):
            raise Exception('volume is not attached to a node')

        pdisk.hotDetach(node.id, volume.id)

        del (volume.extra['node'])

        return True

    def _volume_location(self, volume):
        """
        Recovers the location information from the volume.  If
        the information is not available, then the default
        location for this driver is used.

        """

        try:
            return volume.extra['location']
        except KeyError:
            return self.default_location