예제 #1
0
 def _request_instance(self, zone):
     for i in self.security_group.instances():
         if i.state in ['pending','running'] and i.placement == zone:
             log.info("Using existing instance %s in group %s" % \
                      (i.id,self.security_group.name))
             self._instance = Node(i, self._key_location, 'vol_host')
             break
     if not self._instance:
         log.info("No instance in group %s for zone %s, launching one now." % \
                  (self.security_group.name, zone))
         self._resv = self._ec2.run_instances(image_id=self._image_id,
             instance_type=self._instance_type,
             min_count=1, max_count=1,
             security_groups=[self.security_group.name],
             key_name=self._keypair,
             placement=zone)
         instance = self._resv.instances[0]
         self._instance = Node(instance, self._key_location, 'vol_host')
     s = Spinner()
     log.info("Waiting for instance %s to come up..." % self._instance.id)
     s.start()
     while not self._instance.is_up():
         time.sleep(15)
     s.stop()
     return self._instance
예제 #2
0
 def stream_update_nrm(self):
     for instance in self.instances:
         if instance.id not in self.instances_nrm:
             nrm_cls = partial(NodeRecoveryManager,
                               reboot_interval=self.reboot_interval,
                               n_reboot_restart=self.n_reboot_restart)
             if isinstance(instance, Node):
                 nrm = nrm_cls(instance)
             else:
                 nrm = nrm_cls(Node(instance, self.cluster.key_location))
             self.instances_nrm[instance.id] = nrm
예제 #3
0
 def nodes(self):
     if not self._nodes:
         nodes = self.cluster_group.instances()
         self._nodes = []
         for node in nodes:
             if node.state not in ["pending", "running"]:
                 continue
             alias = self.get_alias(node.id)
             n = Node(node, self.key_location, alias)
             if n.is_master():
                 self._master = n
                 self._nodes.insert(0, n)
             else:
                 self._nodes.append(n)
         # sort the instances by alias?
         # self._nodes.sort(key=lambda n: n.alias)
     else:
         for node in self._nodes:
             log.debug("refreshing instance %s" % node.id)
             node.update()
     return self._nodes
예제 #4
0
 def _request_instance(self, zone):
     alias = self._alias_tmpl % zone
     if self._instance:
         i = self._instance
         if i.state not in ['pending', 'running']:
             raise exception.InstanceNotRunning(i.id)
         if i.placement != zone:
             raise exception.ValidationError(
                 "specified host instance %s is not in zone %s" %
                 (i.id, zone))
         self._instance = Node(i, self._key_location, alias)
     else:
         self._instance = self._get_existing_instance(zone)
     if not self._instance:
         self._validate_image_and_type(self._image_id, self._instance_type)
         log.info(
             "No instance in group %s for zone %s, launching one now." % \
             (self.security_group.name, zone))
         self._resv = self._ec2.run_instances(
             image_id=self._image_id,
             instance_type=self._instance_type,
             min_count=1, max_count=1,
             security_groups=[self.security_group.name],
             key_name=self._keypair,
             placement=zone,
             user_data=alias)
         instance = self._resv.instances[0]
         self._instance = Node(instance, self._key_location, alias)
     s = Spinner()
     log.log(INFO_NO_NEWLINE,
              "Waiting for instance %s to come up..." % self._instance.id)
     s.start()
     while not self._instance.is_up():
         time.sleep(15)
     s.stop()
     return self._instance
예제 #5
0
 def test_filter_etc_hosts_lines(self):
     n = FooNode("node001.starcluster.com", "1.2.3.4")
     lines = [
         "1.2.3.44 master", "1.2.3.4\tnode002", "1.2.3.9 node001",
         "1.2.3.444 node003", "3.4.5.6 node004", "11.2.3.4 node005"
     ]
     kept, rejected = Node.filter_etc_hosts_lines([n], lines)
     assert len(kept) == 4
     assert kept[0] == lines[0]
     assert kept[1] == lines[3]
     assert kept[2] == lines[4]
     assert kept[3] == lines[5]
     assert len(rejected) == 2
     assert rejected[0] == lines[1]
     assert rejected[1] == lines[2]
예제 #6
0
 def test_filter_etc_hosts_lines(self):
     n = FooNode("node001.starcluster.com", "1.2.3.4")
     lines = [
         "1.2.3.44 master",
         "1.2.3.4\tnode002",
         "1.2.3.9 node001",
         "1.2.3.444 node003",
         "3.4.5.6 node004",
         "11.2.3.4 node005",
     ]
     kept, rejected = Node.filter_etc_hosts_lines([n], lines)
     assert len(kept) == 4
     assert kept[0] == lines[0]
     assert kept[1] == lines[3]
     assert kept[2] == lines[4]
     assert kept[3] == lines[5]
     assert len(rejected) == 2
     assert rejected[0] == lines[1]
     assert rejected[1] == lines[2]
예제 #7
0
class VolumeCreator(object):
    def __init__(self, cfg, add_to_cfg=False, keypair=None, device='/dev/sdz',
                 image_id=static.BASE_AMI_32, instance_type="m1.small",
                 shutdown_instance=True):
        self._cfg = cfg
        self._ec2 = cfg.get_easy_ec2()
        self._keypair = keypair
        self._key_location = None
        self._add_to_cfg = add_to_cfg
        self._resv = None
        self._instance = None
        self._volume = None
        self._device = device or '/dev/sdz'
        self._node = None
        self._image_id = image_id or BASE_AMI_32
        self._instance_type = instance_type or 'm1.small'
        self._shutdown = shutdown_instance
        self._security_group = None

    @property
    def security_group(self):
        if not self._security_group:
            self._security_group = self._ec2.get_or_create_group(static.VOLUME_GROUP,
                                               static.VOLUME_GROUP_DESCRIPTION,
                                               auth_ssh=True)
        return self._security_group

    def _request_instance(self, zone):
        for i in self.security_group.instances():
            if i.state in ['pending','running'] and i.placement == zone:
                log.info("Using existing instance %s in group %s" % \
                         (i.id,self.security_group.name))
                self._instance = Node(i, self._key_location, 'vol_host')
                break
        if not self._instance:
            log.info("No instance in group %s for zone %s, launching one now." % \
                     (self.security_group.name, zone))
            self._resv = self._ec2.run_instances(image_id=self._image_id,
                instance_type=self._instance_type,
                min_count=1, max_count=1,
                security_groups=[self.security_group.name],
                key_name=self._keypair,
                placement=zone)
            instance = self._resv.instances[0]
            self._instance = Node(instance, self._key_location, 'vol_host')
        s = Spinner()
        log.info("Waiting for instance %s to come up..." % self._instance.id)
        s.start()
        while not self._instance.is_up():
            time.sleep(15)
        s.stop()
        return self._instance

    def _create_volume(self, size, zone):
        vol = self._ec2.conn.create_volume(size, zone)
        while vol.status != 'available':
            time.sleep(5)
            vol.update()
        self._volume = vol
        return self._volume

    def _determine_device(self):
        block_dev_map = self._instance.block_device_mapping
        for char in string.lowercase[::-1]:
            dev = '/dev/sd%s' % char
            if not block_dev_map.get(dev):
                self._device = dev
                return self._device

    def _attach_volume(self, instance_id, device):
        vol = self._volume
        vol.attach(instance_id, device)
        while True:
            vol.update()
            if vol.attachment_state() == 'attached':
                break
            time.sleep(5)
        return self._volume

    def _validate_image(self, image):
        i = self._ec2.get_image(image)
        if not i or i.id != image:
            raise exception.ValidationError(
                'image %s does not exist' % image
            )

    def _validate_zone(self, zone):
        z = self._ec2.get_zone(zone)
        if not z:
            raise exception.ValidationError(
                'zone %s does not exist' % zone
            )
        if z.state != 'available':
            log.warn('zone %s is not available at this time' % zone)
        return True

    def _validate_size(self, size):
        try:
            volume_size = int(size)
            if volume_size < 1:
                raise exception.ValidationError(
                    "volume_size must be an integer >= 1")
        except ValueError:
            raise exception.ValidationError("volume_size must be an integer")

    def _validate_device(self, device):
        if not utils.is_valid_device(device):
            raise exception.ValidationError("volume device %s is not valid" % \
                                            device)

    def validate(self, size, zone, device, image):
        self._validate_size(size)
        self._validate_zone(zone)
        self._validate_device(device)
        self._validate_image(image)

    def is_valid(self, size, zone, device, image):
        try:
            self.validate(size, zone, device, image)
            return True
        except exception.ValidationError,e:
            log.error(e.msg)
            return False
예제 #8
0
class VolumeCreator(object):
    """
    Handles creating, partitioning, and formatting a new EBS volume.
    By default this class will format the entire drive (without partitioning)
    using the ext3 filesystem.

    host_instance - EC2 instance to use when formatting volume. must exist in
    the same zone as the new volume. if not specified this class will look for
    host instances in the @sc-volumecreator security group.  If it can't find
    an instance in the @sc-volumecreator group that matches the zone of the
    new volume, a new instance is launched.

    shutdown_instance - True will shutdown the host instance after volume
    creation
    """
    def __init__(self, ec2_conn, keypair=None, key_location=None,
                 host_instance=None, device='/dev/sdz',
                 image_id=static.BASE_AMI_32, instance_type="m1.small",
                 shutdown_instance=False, mkfs_cmd='mkfs.ext3',
                 resizefs_cmd='resize2fs'):
        self._ec2 = ec2_conn
        self._keypair = keypair
        self._key_location = key_location
        self._resv = None
        self._instance = host_instance
        self._volume = None
        self._device = device or '/dev/sdz'
        self._node = None
        self._image_id = image_id or static.BASE_AMI_32
        self._instance_type = instance_type or 'm1.small'
        self._shutdown = shutdown_instance
        self._security_group = None
        self._mkfs_cmd = mkfs_cmd
        self._resizefs_cmd = resizefs_cmd
        self._alias_tmpl = "volumecreator-%s"

    def __repr__(self):
        return "<VolumeCreator: %s>" % self._mkfs_cmd

    def __getstate__(self):
        return {}

    @property
    def security_group(self):
        if not self._security_group:
            sg = self._ec2.get_or_create_group(static.VOLUME_GROUP,
                                               cPickle.dumps(self),
                                               auth_ssh=True)
            self._security_group = sg
        return self._security_group

    def _get_existing_instance(self, zone):
        """
        Returns any existing instance in the @sc-volumecreator group that's
        located in zone
        """
        alias = self._alias_tmpl % zone
        sg = self._ec2.get_group_or_none(static.VOLUME_GROUP)
        if not sg:
            return
        for i in sg.instances():
            if i.state in ['pending', 'running'] and i.placement == zone:
                log.info("Using existing instance %s in group %s" % \
                         (i.id, sg.name))
                return Node(i, self._key_location, alias)

    def _request_instance(self, zone):
        alias = self._alias_tmpl % zone
        if self._instance:
            i = self._instance
            if i.state not in ['pending', 'running']:
                raise exception.InstanceNotRunning(i.id)
            if i.placement != zone:
                raise exception.ValidationError(
                    "specified host instance %s is not in zone %s" %
                    (i.id, zone))
            self._instance = Node(i, self._key_location, alias)
        else:
            self._instance = self._get_existing_instance(zone)
        if not self._instance:
            self._validate_image_and_type(self._image_id, self._instance_type)
            log.info(
                "No instance in group %s for zone %s, launching one now." % \
                (self.security_group.name, zone))
            self._resv = self._ec2.run_instances(
                image_id=self._image_id,
                instance_type=self._instance_type,
                min_count=1, max_count=1,
                security_groups=[self.security_group.name],
                key_name=self._keypair,
                placement=zone,
                user_data=alias)
            instance = self._resv.instances[0]
            self._instance = Node(instance, self._key_location, alias)
        s = Spinner()
        log.log(INFO_NO_NEWLINE,
                 "Waiting for instance %s to come up..." % self._instance.id)
        s.start()
        while not self._instance.is_up():
            time.sleep(15)
        s.stop()
        return self._instance

    def _create_volume(self, size, zone, snapshot_id=None):
        msg = "Creating %sGB volume in zone %s" % (size, zone)
        if snapshot_id:
            msg += " from snapshot %s" % snapshot_id
        log.info(msg)
        vol = self._ec2.create_volume(size, zone, snapshot_id)
        log.info("New volume id: %s" % vol.id)
        while vol.status != 'available':
            time.sleep(5)
            vol.update()
        self._volume = vol
        return self._volume

    def _determine_device(self):
        block_dev_map = self._instance.block_device_mapping
        for char in string.lowercase[::-1]:
            dev = '/dev/sd%s' % char
            if not block_dev_map.get(dev):
                self._device = dev
                return self._device

    def _attach_volume(self, vol, instance_id, device):
        log.info("Attaching volume %s to instance %s..." % (vol.id,
                                                            instance_id))
        vol.attach(instance_id, device)
        while True:
            vol.update()
            if vol.attachment_state() == 'attached':
                break
            time.sleep(5)
        return self._volume

    def _validate_image_and_type(self, image, itype):
        img = self._ec2.get_image_or_none(image)
        if not img:
            raise exception.ValidationError(
                'image %s does not exist' % image)
        if not itype in static.INSTANCE_TYPES:
            choices = ', '.join(static.INSTANCE_TYPES)
            raise exception.ValidationError(
                'instance_type must be one of: %s' % choices)
        itype_platform = static.INSTANCE_TYPES.get(itype)
        img_platform = img.architecture
        if img_platform not in itype_platform:
            error_msg = "instance_type %(itype)s is for an " + \
                          "%(iplat)s platform while image_id " + \
                          "%(img)s is an %(imgplat)s platform"
            error_dict = {'itype': itype, 'iplat': ', '.join(itype_platform),
                          'img': img.id, 'imgplat': img_platform}
            raise exception.ValidationError(error_msg % error_dict)

    def _validate_zone(self, zone):
        z = self._ec2.get_zone(zone)
        if not z:
            raise exception.ValidationError(
                'zone %s does not exist' % zone)
        if z.state != 'available':
            log.warn('zone %s is not available at this time' % zone)
        return True

    def _validate_size(self, size):
        try:
            volume_size = int(size)
            if volume_size < 1:
                raise exception.ValidationError(
                    "volume_size must be an integer >= 1")
        except ValueError:
            raise exception.ValidationError("volume_size must be an integer")

    def _validate_device(self, device):
        if not utils.is_valid_device(device):
            raise exception.ValidationError("volume device %s is not valid" % \
                                            device)

    def _validate_required_progs(self, progs):
        log.info("Checking for required remote commands...")
        self._instance.ssh.check_required(progs)

    def validate(self, size, zone, device):
        self._validate_size(size)
        self._validate_zone(zone)
        self._validate_device(device)

    def is_valid(self, size, zone, device):
        try:
            self.validate(size, zone, device)
            return True
        except exception.ValidationError, e:
            log.error(e.msg)
            return False