def detach_volume_group(self, volume_group_id, force=False):
self.db.execute("select "
"instance_id, "
"host, "
"mount_point, "
"v.availability_zone, "
"vg.group_type, "
"vg.block_device, "
"v.block_device, "
"volume_id "
"from volume_groups vg "
"join volumes v using(volume_group_id) "
"left join host_volumes hv using(volume_group_id) "
"left join hosts h using(instance_id) "
"where volume_group_id=%s", (volume_group_id,))
voldata = self.db.fetchall()
if not voldata:
raise VolumeGroupNotFound("Volume group {0} not found".format(volume_group_id))
instance_id, host, mount_point, availability_zone, volume_type, block_device = voldata[0][:6]
if mount_point and not force:
raise VolumeMountError("Volume group {0} is currently mounted on {1}, not detaching".format(volume_group_id, mount_point))
if not host and not force:
raise VolumeNotAvailable("Volume group {0} does not appear to be attached, use force option to force the detachment")
if volume_type == 'raid' and host:
sh = SSHManager()
sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
command = 'mdadm --stop {0}'.format(block_device)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise VolumeMountError("Error stopping the software raid on volume group {0} with command: {1}\n{2}".format(volume_group_id, command, stderr))
volids = []
for d in voldata:
volids.append(d[7])
region = availability_zone[0:len(availability_zone) - 1]
botoconn = self.__get_boto_conn(region)
vols = botoconn.get_all_volumes(volids)
success = True
for vol in vols:
if vol.status == 'in-use':
detached = vol.detach(force)
if not detached:
success = False
if success:
self.configure_volume_automount(volume_group_id, None, True)
self.db.execute("delete from host_volumes where volume_group_id=%s", (volume_group_id, ))
self.dbconn.commit();
print "Volume group {0} detached from instance {1}".format(volume_group_id, instance_id)
else:
print "Volume group {0} not detached".format(volume_group_id)
def unmount_volume_group(self, volume_group_id):
self.db.execute("select "
"hv.mount_point, "
"host, "
"hv.instance_id, "
"h.availability_zone, "
"vg.block_device, "
"vg.group_type, "
"vg.fs_type "
"from host_volumes hv "
"join hosts h on h.instance_id=hv.instance_id "
"join volume_groups vg on vg.volume_group_id=hv.volume_group_id "
"where hv.volume_group_id=%s", (volume_group_id, ))
data = self.db.fetchone()
if not data:
raise VolumeGroupNotFound("Record for volume group {0} not found".format(volume_group_id))
cur_mount_point, host, instance_id, availability_zone, block_device, volume_group_type, fs_type = data
sh = SSHManager()
sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
#TODO check if mounted, if not then noop
block_device_match_pattern = '^([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*'
stdout, stderr, exit_code = sh.sudo('cat /etc/mtab', sudo_password=self.settings.SUDO_PASSWORD)
mtab = stdout.split("\n")
block_device_to_unmount = None
for line in mtab:
m = re.match(block_device_match_pattern, line)
if m:
if volume_group_type == 'single':
if m.group(1) == block_device:
block_device_to_unmount = m.group(1)
elif m.group(1).replace('/dev/xvd', '/dev/sd') == block_device:
block_device_to_unmount = m.group(1)
elif m.group(1).replace('/dev/hd', '/dev/sd') == block_device:
block_device_to_unmount = m.group(1)
else:
if m.group(1) == block_device:
block_device_to_unmount = m.group(1)
if block_device_to_unmount:
command = 'umount {0}'.format(block_device_to_unmount)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise VolumeMountError("Error unmounting volume with command: {0}\n{1}".format(command, stderr))
print "Volume group {0} unmounted from host {1} ".format(volume_group_id, host)
else:
print "Volume group {0} is not mounted ".format(volume_group_id)
self.db.execute("UPDATE host_volumes SET mount_point=%s WHERE instance_id=%s AND volume_group_id=%s", (None, instance_id, volume_group_id))
self.dbconn.commit()
def mount_volume_group(self, instance_id, volume_group_id, mount_point=None, automount=True):
#TODO at some point these should probably be configurable
#TODO check that volume group is attached and assembled
mount_options = 'noatime,nodiratime,noauto'
block_device_match_pattern = '^({0})\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*'
self.db.execute("select "
"hv.mount_point, "
"host, "
"h.availability_zone, "
"vg.block_device, "
"vg.group_type, "
"vg.fs_type "
"from host_volumes hv "
"join hosts h on h.instance_id=hv.instance_id "
"join volume_groups vg on vg.volume_group_id=hv.volume_group_id "
"where hv.instance_id=%s and hv.volume_group_id=%s", (instance_id, volume_group_id))
data = self.db.fetchone()
if not data:
raise VolumeGroupNotFound("Instance {0} not found; unable to lookup availability zone or host for instance".format(instance_id))
cur_mount_point, host, availability_zone, block_device, volume_group_type, fs_type = data
region = availability_zone[0:len(availability_zone) - 1]
sh = SSHManager()
sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
if not mount_point:
stdout, stderr, exit_code = sh.sudo('cat /etc/fstab', sudo_password=self.settings.SUDO_PASSWORD)
mtab = stdout.split("\n")
for line in mtab:
m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line)
if m:
mount_point = m.group(2)
break
if not mount_point:
raise VolumeMountError("No mount point defined and none can be determined for volume group".format(volume_group_id))
#TODO mkdir -p of the mount directory
command = "mkdir -p {0}".format(mount_point)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise VolumeMountError("Unable to create mount directory: {0} with error: {1}".format(mount_point, stderr))
command = 'mount {0} {1} -o {2} -t {3}'.format(block_device, mount_point, mount_options, fs_type)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise VolumeMountError("Error mounting volume with command: {0}\n{1}".format(command, stderr))
self.db.execute("UPDATE host_volumes SET mount_point=%s WHERE instance_id=%s AND volume_group_id=%s", (mount_point, instance_id, volume_group_id))
self.dbconn.commit()
print "Volume group {0} mounted on {1} ({2}) at {3}".format(volume_group_id, host, instance_id, mount_point)
#TODO add the entries to to /etc/mdadm.conf so the raid device is initialized on boot
if automount:
self.configure_volume_automount(volume_group_id, mount_point)
def configure_volume_automount(self, volume_group_id, mount_point=None, remove=False):
mount_options = "noatime,nodiratime 0 0"
block_device_match_pattern = '^({0})\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*'
self.db.execute("select "
"hv.mount_point, "
"host, "
"vg.block_device, "
"vg.group_type, "
"vg.fs_type "
"from hosts h "
"join host_volumes hv on h.instance_id=hv.instance_id and hv.volume_group_id=%s "
"join volume_groups vg on vg.volume_group_id=hv.volume_group_id", (volume_group_id, ))
info = self.db.fetchone()
if not info:
raise VolumeMountError("instance_id, volume_group_id, or host_volume association not found")
defined_mount_point, host, block_device, group_type, fs_type = info
if not block_device:
raise VolumeMountError("block device is not set for volume group {0}, check that the volume group is attached".format(volume_group_id))
sh = SSHManager()
sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
if not remove:
if not mount_point:
if defined_mount_point:
mount_point = defined_mount_point
else:
stdout, stderr, exit_code = sh.sudo('cat /etc/mtab', sudo_password=self.settings.SUDO_PASSWORD)
mtab = stdout.split("\n")
for line in mtab:
m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line)
if m:
mount_point = m.group(2)
break
if not mount_point:
raise VolumeMountError("No mount point defined and none can be determined for volume group".format(volume_group_id))
new_fstab_line = "{0} {1} {2} {3}".format(block_device, mount_point, fs_type, mount_options)
stdout, stderr, exit_code = sh.sudo('cat /etc/fstab', sudo_password=self.settings.SUDO_PASSWORD)
# Checking that stdout is not empty is a safety check to make sure that fstab does not get blown away in case there is some issue getting
# current contents of fstab file. Based on an observed bug that effectively renders an instance useless on reboot
# as /dev/pts doesn't get mounted so ssh does not work
if stdout.strip():
fstab = stdout.split("\n")
found = False
for i in range(0, len(fstab)):
line = fstab[i]
m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line)
if m:
if remove:
fstab[i] = ''
else:
fstab[i] = new_fstab_line
found = True
break
if not found and not remove:
fstab.append(new_fstab_line)
stdout, stderr, exit_code = sh.sudo("mv -f /etc/fstab /etc/fstab.prev")
sh.sudo("echo '{0}' > /etc/fstab".format("\n".join(fstab).replace("\n\n", "\n")), sudo_password=self.settings.SUDO_PASSWORD)
sh.sudo("chmod 0644 /etc/fstab", sudo_password=self.settings.SUDO_PASSWORD)
if not remove:
self.db.execute("update host_volumes set mount_point=%s where volume_group_id=%s", (mount_point, volume_group_id))
self.dbconn.commit()
# at this point /etc/fstab is fully configured
# if problems on debian (or other OS's), there may be more steps needed to get mdadm to autostart
# http://superuser.com/questions/287462/how-can-i-make-mdadm-auto-assemble-raid-after-each-boot
if group_type == 'raid':
print "Reading /etc/mdadm.conf"
stdout, stderr, exit_code = sh.sudo("cat /etc/mdadm.conf", sudo_password=self.settings.SUDO_PASSWORD)
conf = stdout.split("\n")
if not remove:
print "Reading current mdadm devices"
stdout, stderr, exit_code = sh.sudo("mdadm --detail --scan ", sudo_password=self.settings.SUDO_PASSWORD)
scan = stdout.split("\n")
mdadm_line = None
for line in scan:
m = re.match('^ARRAY\s+([^\s]+)\s.*', line)
if m:
if m.group(1) == block_device:
mdadm_line = m.group(0)
else:
stdout, stderr, exit_code = sh.sudo("ls -l --color=never {0}".format(m.group(1)) + " | awk '{print $NF}'", sudo_password=self.settings.SUDO_PASSWORD)
if stdout.strip():
if os.path.basename(stdout.strip()) == os.path.basename(block_device):
mdadm_line = m.group(0).replace(m.group(1), block_device)
if not mdadm_line:
raise VolumeMountError("mdadm --detail --scan did not return an mdadm configuration for {0}".format(block_device))
found = False
for i in range(0, len(conf)):
line = conf[i]
m = re.match('^ARRAY\s+([^\s]+)\s.*', line)
if m and m.group(1) == block_device:
if remove:
conf[i] = ''
else:
conf[i] = mdadm_line
found = True
break
if not found and not remove:
conf.append(mdadm_line)
print "Backing up /etc/mdadm.conf to /etc/mdadm.conf.prev"
sh.sudo('mv -f /etc/mdadm.conf /etc/mdadm.conf.prev', sudo_password=self.settings.SUDO_PASSWORD)
print "Writing new /etc/mdadm.conf file"
for line in conf:
if line:
sh.sudo("echo '{0}' >> /etc/mdadm.conf".format(line), sudo_password=self.settings.SUDO_PASSWORD)
def assemble_raid(self, instance_id, volume_group_id, new_raid=False):
#TODO check that the volumes are attached
self.db.execute("SELECT availability_zone, host from hosts where instance_id=%s", (instance_id, ))
data = self.db.fetchone()
if not data:
raise InstanceNotFound("Instance {0} not found; unable to lookup availability zone or host for instance".format(instance_id))
availability_zone, host = data
region = availability_zone[0:len(availability_zone) - 1]
self.db.execute("select "
"vg.raid_level, "
"vg.stripe_block_size, "
"vg.fs_type, "
"vg.group_type, "
"v.volume_id, "
"v.block_device, "
"v.raid_device_id "
"from volume_groups vg join volumes v on vg.volume_group_id = v.volume_group_id "
"where vg.volume_group_id=%s order by raid_device_id", (volume_group_id, ))
voldata = self.db.fetchall()
if not voldata:
raise VolumeGroupNotFound("Metadata not found for volume_group_id: {0}".format(volume_group_id))
sh = SSHManager()
sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
fs_type = voldata[0][2]
if voldata[0][3] == 'raid':
stdout, stderr, exit_code = sh.sudo('ls --color=never /dev/md[0-9]*', sudo_password=self.settings.SUDO_PASSWORD)
d = stdout.split(' ')
current_devices = []
for i in d:
if i: current_devices.append(str(i))
# find an available md* block device that we can use for the raid
md_id = 0
block_device = "/dev/md" + str(md_id)
while block_device in current_devices:
md_id += 1
block_device = "/dev/md" + str(md_id)
devcount = 0
devlist = ''
md_dev_pattern = ''
for row in voldata:
devcount += 1
devlist += row[5] + " "
# /dev/sd
md_dev_pattern = '([a-z]+{0}).*?'.format(row[5][6:]) + md_dev_pattern
md_dev_pattern = '(md[0-9]+).*?' + md_dev_pattern
if new_raid:
raid_level = voldata[0][0]
stripe_block_size = voldata[0][1]
command = 'mdadm --create {0} --level={1} --chunk={2} --raid-devices={3} {4}'.format(block_device, raid_level, stripe_block_size, devcount, devlist)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise RaidError("There was an error creating raid with command:\n{0}\n{1}".format(command, stderr))
command = 'mkfs.{0} {1}'.format(fs_type, block_device)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise RaidError("There was an error creating filesystem with command:\n{0}\n{1}".format(command, stderr))
else:
# find out if the raid was auto assembled as a new md device before trying to assemble it
stdout, stderr, exit_code = sh.sudo('cat /proc/mdstat', sudo_password=self.settings.SUDO_PASSWORD)
mdstat = stdout.split('\n')
dev_found = None
for line in mdstat:
m = re.match(md_dev_pattern, line)
if m:
dev_found = m.group(1)
if dev_found:
print "Waiting 10 seconds to allow raid device to get ready"
time.sleep(10)
block_device = '/dev/' + dev_found
else:
command = 'mdadm --assemble {0} {1}'.format(block_device, devlist)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise RaidError("There was an error creating raid with command:\n{0}\n{1}".format(command, stderr))
else:
block_device = voldata[0][5]
if new_raid:
command = 'mkfs.{0} {1}'.format(fs_type, block_device)
stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
raise RaidError("There was an error creating filesystem with command:\n{0}\n{1}".format(command, stderr))
#TODO add check in here to cat /proc/mdstat and make sure the expected raid is setup
self.db.execute("INSERT INTO host_volumes set instance_id=%s, volume_group_id=%s, mount_point=NULL ON DUPLICATE KEY UPDATE mount_point=NULL", (instance_id, volume_group_id))
self.db.execute("UPDATE volume_groups set block_device=%s where volume_group_id=%s", (block_device, volume_group_id))
self.dbconn.commit()
def configure_hostname(self, instance_id, hostname, configure_server=False):
self.db.execute("select instance_id, hostname_external, hostname_internal from hosts where host=%s",(hostname, ))
rows = self.db.fetchall()
# This updates any hosts with the hostname given to their external or internal hostname before applying that hostname to another host
if rows:
for row in rows:
hn = None
if row[2]:
hn = row[2]
if row[1]:
hn = row[1]
self.db.execute("update hosts set host=%s where instance_id=%s", (hn, row[0]))
self.dbconn.commit()
self.db.execute("update hosts set host=%s where instance_id=%s", (hostname, instance_id))
self.dbconn.commit()
self.db.execute("select instance_id, host, uname from hosts where instance_id=%s", (instance_id, ))
row = self.db.fetchone()
if not row:
self.logger.error("Unable to find instance metadata")
return
# if there is a uname set then we will use that rather than the hostname to set the system uname
uname = row[1]
if row[2]:
uname = row[2]
sh = SSHManager()
sh.connect(hostname=hostname, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE)
self.logger.info("Setting the running value for hostname on the instance")
stdout, stderr, exit_code = sh.sudo('hostname {0}'.format(uname), sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) != 0:
self.logger.error("There was an error setting the running hostname of the instance\n" + stderr)
return
permanent = False
# Redhat/CentOS uses a "HOSTNAME=somehost.example.com" line in /etc/sysconfig/network to set hostname permanently
stdout, stderr, exit_code = sh.sudo('cat /etc/sysconfig/network', sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) == 0:
self.logger.info("/etc/sysconfig/network file found, modifying HOSTNAME")
hoststring = "HOSTNAME={0}".format(uname)
lines = stdout.strip().split("\n")
found = False
for i in range(0, len(lines)):
if lines[i][0:8] == 'HOSTNAME':
lines[i] = hoststring
found = True
break
if not found:
lines.append(hoststring)
sh.sudo('mv -f /etc/sysconfig/network /etc/sysconfig/network.prev', sudo_password=self.settings.SUDO_PASSWORD)
for line in lines:
sh.sudo('echo {0} >> /etc/sysconfig/network'.format(line), sudo_password=self.settings.SUDO_PASSWORD)
permanent = True
# Ubuntu uses "somehost.example.com" as the contents of /etc/hostname to set hostname permanently
stdout, stderr, exit_code = sh.sudo('cat /etc/hostname', sudo_password=self.settings.SUDO_PASSWORD)
if int(exit_code) == 0:
self.logger.info("/etc/hostname file found, setting hostname")
sh.sudo('cp /etc/hostname /etc/hostname.prev', sudo_password=self.settings.SUDO_PASSWORD)
sh.sudo('echo {0} > /etc/hostname'.format(uname), sudo_password=self.settings.SUDO_PASSWORD)
permanent = True
if permanent:
self.logger.info("Hostname configured permanently on instance")
