def detach_volume_group(self, volume_group_id, force=False): self.db.execute("select " "instance_id, " "host, " "mount_point, " "v.availability_zone, " "vg.group_type, " "vg.block_device, " "v.block_device, " "volume_id " "from volume_groups vg " "join volumes v using(volume_group_id) " "left join host_volumes hv using(volume_group_id) " "left join hosts h using(instance_id) " "where volume_group_id=%s", (volume_group_id,)) voldata = self.db.fetchall() if not voldata: raise VolumeGroupNotFound("Volume group {0} not found".format(volume_group_id)) instance_id, host, mount_point, availability_zone, volume_type, block_device = voldata[0][:6] if mount_point and not force: raise VolumeMountError("Volume group {0} is currently mounted on {1}, not detaching".format(volume_group_id, mount_point)) if not host and not force: raise VolumeNotAvailable("Volume group {0} does not appear to be attached, use force option to force the detachment") if volume_type == 'raid' and host: sh = SSHManager() sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) command = 'mdadm --stop {0}'.format(block_device) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise VolumeMountError("Error stopping the software raid on volume group {0} with command: {1}\n{2}".format(volume_group_id, command, stderr)) volids = [] for d in voldata: volids.append(d[7]) region = availability_zone[0:len(availability_zone) - 1] botoconn = self.__get_boto_conn(region) vols = botoconn.get_all_volumes(volids) success = True for vol in vols: if vol.status == 'in-use': detached = vol.detach(force) if not detached: success = False if success: self.configure_volume_automount(volume_group_id, None, True) self.db.execute("delete from host_volumes where volume_group_id=%s", (volume_group_id, )) self.dbconn.commit(); print "Volume group {0} detached from instance {1}".format(volume_group_id, instance_id) else: print "Volume group {0} not detached".format(volume_group_id)
def mount_volume_group(self, instance_id, volume_group_id, mount_point=None, automount=True): #TODO at some point these should probably be configurable #TODO check that volume group is attached and assembled mount_options = 'noatime,nodiratime,noauto' block_device_match_pattern = '^({0})\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*' self.db.execute("select " "hv.mount_point, " "host, " "h.availability_zone, " "vg.block_device, " "vg.group_type, " "vg.fs_type " "from host_volumes hv " "join hosts h on h.instance_id=hv.instance_id " "join volume_groups vg on vg.volume_group_id=hv.volume_group_id " "where hv.instance_id=%s and hv.volume_group_id=%s", (instance_id, volume_group_id)) data = self.db.fetchone() if not data: raise VolumeGroupNotFound("Instance {0} not found; unable to lookup availability zone or host for instance".format(instance_id)) cur_mount_point, host, availability_zone, block_device, volume_group_type, fs_type = data region = availability_zone[0:len(availability_zone) - 1] sh = SSHManager() sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) if not mount_point: stdout, stderr, exit_code = sh.sudo('cat /etc/fstab', sudo_password=self.settings.SUDO_PASSWORD) mtab = stdout.split("\n") for line in mtab: m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line) if m: mount_point = m.group(2) break if not mount_point: raise VolumeMountError("No mount point defined and none can be determined for volume group".format(volume_group_id)) #TODO mkdir -p of the mount directory command = "mkdir -p {0}".format(mount_point) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise VolumeMountError("Unable to create mount directory: {0} with error: {1}".format(mount_point, stderr)) command = 'mount {0} {1} -o {2} -t {3}'.format(block_device, mount_point, mount_options, fs_type) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise VolumeMountError("Error mounting volume with command: {0}\n{1}".format(command, stderr)) self.db.execute("UPDATE host_volumes SET mount_point=%s WHERE instance_id=%s AND volume_group_id=%s", (mount_point, instance_id, volume_group_id)) self.dbconn.commit() print "Volume group {0} mounted on {1} ({2}) at {3}".format(volume_group_id, host, instance_id, mount_point) #TODO add the entries to to /etc/mdadm.conf so the raid device is initialized on boot if automount: self.configure_volume_automount(volume_group_id, mount_point)
def unmount_volume_group(self, volume_group_id): self.db.execute("select " "hv.mount_point, " "host, " "hv.instance_id, " "h.availability_zone, " "vg.block_device, " "vg.group_type, " "vg.fs_type " "from host_volumes hv " "join hosts h on h.instance_id=hv.instance_id " "join volume_groups vg on vg.volume_group_id=hv.volume_group_id " "where hv.volume_group_id=%s", (volume_group_id, )) data = self.db.fetchone() if not data: raise VolumeGroupNotFound("Record for volume group {0} not found".format(volume_group_id)) cur_mount_point, host, instance_id, availability_zone, block_device, volume_group_type, fs_type = data sh = SSHManager() sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) #TODO check if mounted, if not then noop block_device_match_pattern = '^([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*' stdout, stderr, exit_code = sh.sudo('cat /etc/mtab', sudo_password=self.settings.SUDO_PASSWORD) mtab = stdout.split("\n") block_device_to_unmount = None for line in mtab: m = re.match(block_device_match_pattern, line) if m: if volume_group_type == 'single': if m.group(1) == block_device: block_device_to_unmount = m.group(1) elif m.group(1).replace('/dev/xvd', '/dev/sd') == block_device: block_device_to_unmount = m.group(1) elif m.group(1).replace('/dev/hd', '/dev/sd') == block_device: block_device_to_unmount = m.group(1) else: if m.group(1) == block_device: block_device_to_unmount = m.group(1) if block_device_to_unmount: command = 'umount {0}'.format(block_device_to_unmount) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise VolumeMountError("Error unmounting volume with command: {0}\n{1}".format(command, stderr)) print "Volume group {0} unmounted from host {1} ".format(volume_group_id, host) else: print "Volume group {0} is not mounted ".format(volume_group_id) self.db.execute("UPDATE host_volumes SET mount_point=%s WHERE instance_id=%s AND volume_group_id=%s", (None, instance_id, volume_group_id)) self.dbconn.commit()
def configure_volume_automount(self, volume_group_id, mount_point=None, remove=False): mount_options = "noatime,nodiratime 0 0" block_device_match_pattern = '^({0})\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+([0-9])\s+([0-9]).*' self.db.execute("select " "hv.mount_point, " "host, " "vg.block_device, " "vg.group_type, " "vg.fs_type " "from hosts h " "join host_volumes hv on h.instance_id=hv.instance_id and hv.volume_group_id=%s " "join volume_groups vg on vg.volume_group_id=hv.volume_group_id", (volume_group_id, )) info = self.db.fetchone() if not info: raise VolumeMountError("instance_id, volume_group_id, or host_volume association not found") defined_mount_point, host, block_device, group_type, fs_type = info if not block_device: raise VolumeMountError("block device is not set for volume group {0}, check that the volume group is attached".format(volume_group_id)) sh = SSHManager() sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) if not remove: if not mount_point: if defined_mount_point: mount_point = defined_mount_point else: stdout, stderr, exit_code = sh.sudo('cat /etc/mtab', sudo_password=self.settings.SUDO_PASSWORD) mtab = stdout.split("\n") for line in mtab: m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line) if m: mount_point = m.group(2) break if not mount_point: raise VolumeMountError("No mount point defined and none can be determined for volume group".format(volume_group_id)) new_fstab_line = "{0} {1} {2} {3}".format(block_device, mount_point, fs_type, mount_options) stdout, stderr, exit_code = sh.sudo('cat /etc/fstab', sudo_password=self.settings.SUDO_PASSWORD) # Checking that stdout is not empty is a safety check to make sure that fstab does not get blown away in case there is some issue getting # current contents of fstab file. Based on an observed bug that effectively renders an instance useless on reboot # as /dev/pts doesn't get mounted so ssh does not work if stdout.strip(): fstab = stdout.split("\n") found = False for i in range(0, len(fstab)): line = fstab[i] m = re.match(block_device_match_pattern.format(block_device.replace('/', '\\/')), line) if m: if remove: fstab[i] = '' else: fstab[i] = new_fstab_line found = True break if not found and not remove: fstab.append(new_fstab_line) stdout, stderr, exit_code = sh.sudo("mv -f /etc/fstab /etc/fstab.prev") sh.sudo("echo '{0}' > /etc/fstab".format("\n".join(fstab).replace("\n\n", "\n")), sudo_password=self.settings.SUDO_PASSWORD) sh.sudo("chmod 0644 /etc/fstab", sudo_password=self.settings.SUDO_PASSWORD) if not remove: self.db.execute("update host_volumes set mount_point=%s where volume_group_id=%s", (mount_point, volume_group_id)) self.dbconn.commit() # at this point /etc/fstab is fully configured # if problems on debian (or other OS's), there may be more steps needed to get mdadm to autostart # http://superuser.com/questions/287462/how-can-i-make-mdadm-auto-assemble-raid-after-each-boot if group_type == 'raid': print "Reading /etc/mdadm.conf" stdout, stderr, exit_code = sh.sudo("cat /etc/mdadm.conf", sudo_password=self.settings.SUDO_PASSWORD) conf = stdout.split("\n") if not remove: print "Reading current mdadm devices" stdout, stderr, exit_code = sh.sudo("mdadm --detail --scan ", sudo_password=self.settings.SUDO_PASSWORD) scan = stdout.split("\n") mdadm_line = None for line in scan: m = re.match('^ARRAY\s+([^\s]+)\s.*', line) if m: if m.group(1) == block_device: mdadm_line = m.group(0) else: stdout, stderr, exit_code = sh.sudo("ls -l --color=never {0}".format(m.group(1)) + " | awk '{print $NF}'", sudo_password=self.settings.SUDO_PASSWORD) if stdout.strip(): if os.path.basename(stdout.strip()) == os.path.basename(block_device): mdadm_line = m.group(0).replace(m.group(1), block_device) if not mdadm_line: raise VolumeMountError("mdadm --detail --scan did not return an mdadm configuration for {0}".format(block_device)) found = False for i in range(0, len(conf)): line = conf[i] m = re.match('^ARRAY\s+([^\s]+)\s.*', line) if m and m.group(1) == block_device: if remove: conf[i] = '' else: conf[i] = mdadm_line found = True break if not found and not remove: conf.append(mdadm_line) print "Backing up /etc/mdadm.conf to /etc/mdadm.conf.prev" sh.sudo('mv -f /etc/mdadm.conf /etc/mdadm.conf.prev', sudo_password=self.settings.SUDO_PASSWORD) print "Writing new /etc/mdadm.conf file" for line in conf: if line: sh.sudo("echo '{0}' >> /etc/mdadm.conf".format(line), sudo_password=self.settings.SUDO_PASSWORD)
def assemble_raid(self, instance_id, volume_group_id, new_raid=False): #TODO check that the volumes are attached self.db.execute("SELECT availability_zone, host from hosts where instance_id=%s", (instance_id, )) data = self.db.fetchone() if not data: raise InstanceNotFound("Instance {0} not found; unable to lookup availability zone or host for instance".format(instance_id)) availability_zone, host = data region = availability_zone[0:len(availability_zone) - 1] self.db.execute("select " "vg.raid_level, " "vg.stripe_block_size, " "vg.fs_type, " "vg.group_type, " "v.volume_id, " "v.block_device, " "v.raid_device_id " "from volume_groups vg join volumes v on vg.volume_group_id = v.volume_group_id " "where vg.volume_group_id=%s order by raid_device_id", (volume_group_id, )) voldata = self.db.fetchall() if not voldata: raise VolumeGroupNotFound("Metadata not found for volume_group_id: {0}".format(volume_group_id)) sh = SSHManager() sh.connect(hostname=host, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) fs_type = voldata[0][2] if voldata[0][3] == 'raid': stdout, stderr, exit_code = sh.sudo('ls --color=never /dev/md[0-9]*', sudo_password=self.settings.SUDO_PASSWORD) d = stdout.split(' ') current_devices = [] for i in d: if i: current_devices.append(str(i)) # find an available md* block device that we can use for the raid md_id = 0 block_device = "/dev/md" + str(md_id) while block_device in current_devices: md_id += 1 block_device = "/dev/md" + str(md_id) devcount = 0 devlist = '' md_dev_pattern = '' for row in voldata: devcount += 1 devlist += row[5] + " " # /dev/sd md_dev_pattern = '([a-z]+{0}).*?'.format(row[5][6:]) + md_dev_pattern md_dev_pattern = '(md[0-9]+).*?' + md_dev_pattern if new_raid: raid_level = voldata[0][0] stripe_block_size = voldata[0][1] command = 'mdadm --create {0} --level={1} --chunk={2} --raid-devices={3} {4}'.format(block_device, raid_level, stripe_block_size, devcount, devlist) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise RaidError("There was an error creating raid with command:\n{0}\n{1}".format(command, stderr)) command = 'mkfs.{0} {1}'.format(fs_type, block_device) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise RaidError("There was an error creating filesystem with command:\n{0}\n{1}".format(command, stderr)) else: # find out if the raid was auto assembled as a new md device before trying to assemble it stdout, stderr, exit_code = sh.sudo('cat /proc/mdstat', sudo_password=self.settings.SUDO_PASSWORD) mdstat = stdout.split('\n') dev_found = None for line in mdstat: m = re.match(md_dev_pattern, line) if m: dev_found = m.group(1) if dev_found: print "Waiting 10 seconds to allow raid device to get ready" time.sleep(10) block_device = '/dev/' + dev_found else: command = 'mdadm --assemble {0} {1}'.format(block_device, devlist) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise RaidError("There was an error creating raid with command:\n{0}\n{1}".format(command, stderr)) else: block_device = voldata[0][5] if new_raid: command = 'mkfs.{0} {1}'.format(fs_type, block_device) stdout, stderr, exit_code = sh.sudo(command=command, sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: raise RaidError("There was an error creating filesystem with command:\n{0}\n{1}".format(command, stderr)) #TODO add check in here to cat /proc/mdstat and make sure the expected raid is setup self.db.execute("INSERT INTO host_volumes set instance_id=%s, volume_group_id=%s, mount_point=NULL ON DUPLICATE KEY UPDATE mount_point=NULL", (instance_id, volume_group_id)) self.db.execute("UPDATE volume_groups set block_device=%s where volume_group_id=%s", (block_device, volume_group_id)) self.dbconn.commit()
def configure_hostname(self, instance_id, hostname, configure_server=False): self.db.execute("select instance_id, hostname_external, hostname_internal from hosts where host=%s",(hostname, )) rows = self.db.fetchall() # This updates any hosts with the hostname given to their external or internal hostname before applying that hostname to another host if rows: for row in rows: hn = None if row[2]: hn = row[2] if row[1]: hn = row[1] self.db.execute("update hosts set host=%s where instance_id=%s", (hn, row[0])) self.dbconn.commit() self.db.execute("update hosts set host=%s where instance_id=%s", (hostname, instance_id)) self.dbconn.commit() self.db.execute("select instance_id, host, uname from hosts where instance_id=%s", (instance_id, )) row = self.db.fetchone() if not row: self.logger.error("Unable to find instance metadata") return # if there is a uname set then we will use that rather than the hostname to set the system uname uname = row[1] if row[2]: uname = row[2] sh = SSHManager() sh.connect(hostname=hostname, port=self.settings.SSH_PORT, username=self.settings.SSH_USER, password=self.settings.SSH_PASSWORD, key_filename=self.settings.SSH_KEYFILE) self.logger.info("Setting the running value for hostname on the instance") stdout, stderr, exit_code = sh.sudo('hostname {0}'.format(uname), sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) != 0: self.logger.error("There was an error setting the running hostname of the instance\n" + stderr) return permanent = False # Redhat/CentOS uses a "HOSTNAME=somehost.example.com" line in /etc/sysconfig/network to set hostname permanently stdout, stderr, exit_code = sh.sudo('cat /etc/sysconfig/network', sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) == 0: self.logger.info("/etc/sysconfig/network file found, modifying HOSTNAME") hoststring = "HOSTNAME={0}".format(uname) lines = stdout.strip().split("\n") found = False for i in range(0, len(lines)): if lines[i][0:8] == 'HOSTNAME': lines[i] = hoststring found = True break if not found: lines.append(hoststring) sh.sudo('mv -f /etc/sysconfig/network /etc/sysconfig/network.prev', sudo_password=self.settings.SUDO_PASSWORD) for line in lines: sh.sudo('echo {0} >> /etc/sysconfig/network'.format(line), sudo_password=self.settings.SUDO_PASSWORD) permanent = True # Ubuntu uses "somehost.example.com" as the contents of /etc/hostname to set hostname permanently stdout, stderr, exit_code = sh.sudo('cat /etc/hostname', sudo_password=self.settings.SUDO_PASSWORD) if int(exit_code) == 0: self.logger.info("/etc/hostname file found, setting hostname") sh.sudo('cp /etc/hostname /etc/hostname.prev', sudo_password=self.settings.SUDO_PASSWORD) sh.sudo('echo {0} > /etc/hostname'.format(uname), sudo_password=self.settings.SUDO_PASSWORD) permanent = True if permanent: self.logger.info("Hostname configured permanently on instance")