def _request_instance(self, zone):
for i in self.security_group.instances():
if i.state in ['pending','running'] and i.placement == zone:
log.info("Using existing instance %s in group %s" % \
(i.id,self.security_group.name))
self._instance = Node(i, self._key_location, 'vol_host')
break
if not self._instance:
log.info("No instance in group %s for zone %s, launching one now." % \
(self.security_group.name, zone))
self._resv = self._ec2.run_instances(image_id=self._image_id,
instance_type=self._instance_type,
min_count=1, max_count=1,
security_groups=[self.security_group.name],
key_name=self._keypair,
placement=zone)
instance = self._resv.instances[0]
self._instance = Node(instance, self._key_location, 'vol_host')
s = Spinner()
log.info("Waiting for instance %s to come up..." % self._instance.id)
s.start()
while not self._instance.is_up():
time.sleep(15)
s.stop()
return self._instance
def stream_update_nrm(self):
for instance in self.instances:
if instance.id not in self.instances_nrm:
nrm_cls = partial(NodeRecoveryManager,
reboot_interval=self.reboot_interval,
n_reboot_restart=self.n_reboot_restart)
if isinstance(instance, Node):
nrm = nrm_cls(instance)
else:
nrm = nrm_cls(Node(instance, self.cluster.key_location))
self.instances_nrm[instance.id] = nrm
def nodes(self):
if not self._nodes:
nodes = self.cluster_group.instances()
self._nodes = []
for node in nodes:
if node.state not in ["pending", "running"]:
continue
alias = self.get_alias(node.id)
n = Node(node, self.key_location, alias)
if n.is_master():
self._master = n
self._nodes.insert(0, n)
else:
self._nodes.append(n)
# sort the instances by alias?
# self._nodes.sort(key=lambda n: n.alias)
else:
for node in self._nodes:
log.debug("refreshing instance %s" % node.id)
node.update()
return self._nodes
def _request_instance(self, zone):
alias = self._alias_tmpl % zone
if self._instance:
i = self._instance
if i.state not in ['pending', 'running']:
raise exception.InstanceNotRunning(i.id)
if i.placement != zone:
raise exception.ValidationError(
"specified host instance %s is not in zone %s" %
(i.id, zone))
self._instance = Node(i, self._key_location, alias)
else:
self._instance = self._get_existing_instance(zone)
if not self._instance:
self._validate_image_and_type(self._image_id, self._instance_type)
log.info(
"No instance in group %s for zone %s, launching one now." % \
(self.security_group.name, zone))
self._resv = self._ec2.run_instances(
image_id=self._image_id,
instance_type=self._instance_type,
min_count=1, max_count=1,
security_groups=[self.security_group.name],
key_name=self._keypair,
placement=zone,
user_data=alias)
instance = self._resv.instances[0]
self._instance = Node(instance, self._key_location, alias)
s = Spinner()
log.log(INFO_NO_NEWLINE,
"Waiting for instance %s to come up..." % self._instance.id)
s.start()
while not self._instance.is_up():
time.sleep(15)
s.stop()
return self._instance
def test_filter_etc_hosts_lines(self):
n = FooNode("node001.starcluster.com", "1.2.3.4")
lines = [
"1.2.3.44 master", "1.2.3.4\tnode002", "1.2.3.9 node001",
"1.2.3.444 node003", "3.4.5.6 node004", "11.2.3.4 node005"
]
kept, rejected = Node.filter_etc_hosts_lines([n], lines)
assert len(kept) == 4
assert kept[0] == lines[0]
assert kept[1] == lines[3]
assert kept[2] == lines[4]
assert kept[3] == lines[5]
assert len(rejected) == 2
assert rejected[0] == lines[1]
assert rejected[1] == lines[2]
def test_filter_etc_hosts_lines(self):
n = FooNode("node001.starcluster.com", "1.2.3.4")
lines = [
"1.2.3.44 master",
"1.2.3.4\tnode002",
"1.2.3.9 node001",
"1.2.3.444 node003",
"3.4.5.6 node004",
"11.2.3.4 node005",
]
kept, rejected = Node.filter_etc_hosts_lines([n], lines)
assert len(kept) == 4
assert kept[0] == lines[0]
assert kept[1] == lines[3]
assert kept[2] == lines[4]
assert kept[3] == lines[5]
assert len(rejected) == 2
assert rejected[0] == lines[1]
assert rejected[1] == lines[2]
class VolumeCreator(object):
def __init__(self, cfg, add_to_cfg=False, keypair=None, device='/dev/sdz',
image_id=static.BASE_AMI_32, instance_type="m1.small",
shutdown_instance=True):
self._cfg = cfg
self._ec2 = cfg.get_easy_ec2()
self._keypair = keypair
self._key_location = None
self._add_to_cfg = add_to_cfg
self._resv = None
self._instance = None
self._volume = None
self._device = device or '/dev/sdz'
self._node = None
self._image_id = image_id or BASE_AMI_32
self._instance_type = instance_type or 'm1.small'
self._shutdown = shutdown_instance
self._security_group = None
@property
def security_group(self):
if not self._security_group:
self._security_group = self._ec2.get_or_create_group(static.VOLUME_GROUP,
static.VOLUME_GROUP_DESCRIPTION,
auth_ssh=True)
return self._security_group
def _request_instance(self, zone):
for i in self.security_group.instances():
if i.state in ['pending','running'] and i.placement == zone:
log.info("Using existing instance %s in group %s" % \
(i.id,self.security_group.name))
self._instance = Node(i, self._key_location, 'vol_host')
break
if not self._instance:
log.info("No instance in group %s for zone %s, launching one now." % \
(self.security_group.name, zone))
self._resv = self._ec2.run_instances(image_id=self._image_id,
instance_type=self._instance_type,
min_count=1, max_count=1,
security_groups=[self.security_group.name],
key_name=self._keypair,
placement=zone)
instance = self._resv.instances[0]
self._instance = Node(instance, self._key_location, 'vol_host')
s = Spinner()
log.info("Waiting for instance %s to come up..." % self._instance.id)
s.start()
while not self._instance.is_up():
time.sleep(15)
s.stop()
return self._instance
def _create_volume(self, size, zone):
vol = self._ec2.conn.create_volume(size, zone)
while vol.status != 'available':
time.sleep(5)
vol.update()
self._volume = vol
return self._volume
def _determine_device(self):
block_dev_map = self._instance.block_device_mapping
for char in string.lowercase[::-1]:
dev = '/dev/sd%s' % char
if not block_dev_map.get(dev):
self._device = dev
return self._device
def _attach_volume(self, instance_id, device):
vol = self._volume
vol.attach(instance_id, device)
while True:
vol.update()
if vol.attachment_state() == 'attached':
break
time.sleep(5)
return self._volume
def _validate_image(self, image):
i = self._ec2.get_image(image)
if not i or i.id != image:
raise exception.ValidationError(
'image %s does not exist' % image
)
def _validate_zone(self, zone):
z = self._ec2.get_zone(zone)
if not z:
raise exception.ValidationError(
'zone %s does not exist' % zone
)
if z.state != 'available':
log.warn('zone %s is not available at this time' % zone)
return True
def _validate_size(self, size):
try:
volume_size = int(size)
if volume_size < 1:
raise exception.ValidationError(
"volume_size must be an integer >= 1")
except ValueError:
raise exception.ValidationError("volume_size must be an integer")
def _validate_device(self, device):
if not utils.is_valid_device(device):
raise exception.ValidationError("volume device %s is not valid" % \
device)
def validate(self, size, zone, device, image):
self._validate_size(size)
self._validate_zone(zone)
self._validate_device(device)
self._validate_image(image)
def is_valid(self, size, zone, device, image):
try:
self.validate(size, zone, device, image)
return True
except exception.ValidationError,e:
log.error(e.msg)
return False
class VolumeCreator(object):
"""
Handles creating, partitioning, and formatting a new EBS volume.
By default this class will format the entire drive (without partitioning)
using the ext3 filesystem.
host_instance - EC2 instance to use when formatting volume. must exist in
the same zone as the new volume. if not specified this class will look for
host instances in the @sc-volumecreator security group. If it can't find
an instance in the @sc-volumecreator group that matches the zone of the
new volume, a new instance is launched.
shutdown_instance - True will shutdown the host instance after volume
creation
"""
def __init__(self, ec2_conn, keypair=None, key_location=None,
host_instance=None, device='/dev/sdz',
image_id=static.BASE_AMI_32, instance_type="m1.small",
shutdown_instance=False, mkfs_cmd='mkfs.ext3',
resizefs_cmd='resize2fs'):
self._ec2 = ec2_conn
self._keypair = keypair
self._key_location = key_location
self._resv = None
self._instance = host_instance
self._volume = None
self._device = device or '/dev/sdz'
self._node = None
self._image_id = image_id or static.BASE_AMI_32
self._instance_type = instance_type or 'm1.small'
self._shutdown = shutdown_instance
self._security_group = None
self._mkfs_cmd = mkfs_cmd
self._resizefs_cmd = resizefs_cmd
self._alias_tmpl = "volumecreator-%s"
def __repr__(self):
return "<VolumeCreator: %s>" % self._mkfs_cmd
def __getstate__(self):
return {}
@property
def security_group(self):
if not self._security_group:
sg = self._ec2.get_or_create_group(static.VOLUME_GROUP,
cPickle.dumps(self),
auth_ssh=True)
self._security_group = sg
return self._security_group
def _get_existing_instance(self, zone):
"""
Returns any existing instance in the @sc-volumecreator group that's
located in zone
"""
alias = self._alias_tmpl % zone
sg = self._ec2.get_group_or_none(static.VOLUME_GROUP)
if not sg:
return
for i in sg.instances():
if i.state in ['pending', 'running'] and i.placement == zone:
log.info("Using existing instance %s in group %s" % \
(i.id, sg.name))
return Node(i, self._key_location, alias)
def _request_instance(self, zone):
alias = self._alias_tmpl % zone
if self._instance:
i = self._instance
if i.state not in ['pending', 'running']:
raise exception.InstanceNotRunning(i.id)
if i.placement != zone:
raise exception.ValidationError(
"specified host instance %s is not in zone %s" %
(i.id, zone))
self._instance = Node(i, self._key_location, alias)
else:
self._instance = self._get_existing_instance(zone)
if not self._instance:
self._validate_image_and_type(self._image_id, self._instance_type)
log.info(
"No instance in group %s for zone %s, launching one now." % \
(self.security_group.name, zone))
self._resv = self._ec2.run_instances(
image_id=self._image_id,
instance_type=self._instance_type,
min_count=1, max_count=1,
security_groups=[self.security_group.name],
key_name=self._keypair,
placement=zone,
user_data=alias)
instance = self._resv.instances[0]
self._instance = Node(instance, self._key_location, alias)
s = Spinner()
log.log(INFO_NO_NEWLINE,
"Waiting for instance %s to come up..." % self._instance.id)
s.start()
while not self._instance.is_up():
time.sleep(15)
s.stop()
return self._instance
def _create_volume(self, size, zone, snapshot_id=None):
msg = "Creating %sGB volume in zone %s" % (size, zone)
if snapshot_id:
msg += " from snapshot %s" % snapshot_id
log.info(msg)
vol = self._ec2.create_volume(size, zone, snapshot_id)
log.info("New volume id: %s" % vol.id)
while vol.status != 'available':
time.sleep(5)
vol.update()
self._volume = vol
return self._volume
def _determine_device(self):
block_dev_map = self._instance.block_device_mapping
for char in string.lowercase[::-1]:
dev = '/dev/sd%s' % char
if not block_dev_map.get(dev):
self._device = dev
return self._device
def _attach_volume(self, vol, instance_id, device):
log.info("Attaching volume %s to instance %s..." % (vol.id,
instance_id))
vol.attach(instance_id, device)
while True:
vol.update()
if vol.attachment_state() == 'attached':
break
time.sleep(5)
return self._volume
def _validate_image_and_type(self, image, itype):
img = self._ec2.get_image_or_none(image)
if not img:
raise exception.ValidationError(
'image %s does not exist' % image)
if not itype in static.INSTANCE_TYPES:
choices = ', '.join(static.INSTANCE_TYPES)
raise exception.ValidationError(
'instance_type must be one of: %s' % choices)
itype_platform = static.INSTANCE_TYPES.get(itype)
img_platform = img.architecture
if img_platform not in itype_platform:
error_msg = "instance_type %(itype)s is for an " + \
"%(iplat)s platform while image_id " + \
"%(img)s is an %(imgplat)s platform"
error_dict = {'itype': itype, 'iplat': ', '.join(itype_platform),
'img': img.id, 'imgplat': img_platform}
raise exception.ValidationError(error_msg % error_dict)
def _validate_zone(self, zone):
z = self._ec2.get_zone(zone)
if not z:
raise exception.ValidationError(
'zone %s does not exist' % zone)
if z.state != 'available':
log.warn('zone %s is not available at this time' % zone)
return True
def _validate_size(self, size):
try:
volume_size = int(size)
if volume_size < 1:
raise exception.ValidationError(
"volume_size must be an integer >= 1")
except ValueError:
raise exception.ValidationError("volume_size must be an integer")
def _validate_device(self, device):
if not utils.is_valid_device(device):
raise exception.ValidationError("volume device %s is not valid" % \
device)
def _validate_required_progs(self, progs):
log.info("Checking for required remote commands...")
self._instance.ssh.check_required(progs)
def validate(self, size, zone, device):
self._validate_size(size)
self._validate_zone(zone)
self._validate_device(device)
def is_valid(self, size, zone, device):
try:
self.validate(size, zone, device)
return True
except exception.ValidationError, e:
log.error(e.msg)
return False