def mount_ex(device, mpoint='/mnt', options=None, make_fs=False, fstype='ext3', auto_mount=False): if not os.path.exists(mpoint): os.makedirs(mpoint) options = options or ('-t', 'auto') if make_fs: from scalarizr import storage2 storage2.filesystem(fstype).mkfs(device) out = mount(device, mpoint, *options)[0] if " ".join(options).find("loop") == -1: mtab = mounts() if not mtab.contains(device): raise MountError("Cannot mount device '%s'. %s" % (device, out), MountError.CANNOT_MOUNT) if auto_mount: _fstab = fstab() if not _fstab.contains(device, mpoint=mpoint, reload=True): opts = "defaults" if linux.os.ubuntu and linux.os['version'] >= (10, 4): opts += ',comment=cloudconfig,nobootwait' _fstab.append(device, mpoint, options=opts)
def _format_image(self): LOG.info("Formatting image") vol_entry = [v for v in self._mtab if v.device.startswith('/dev')][0] if vol_entry.device == '/dev/root' and not os.path.exists( vol_entry.device): vol_entry = [ v for v in mount.mounts('/etc/mtab') if v.device.startswith('/dev') ][0] fs = filesystem(vol_entry.fstype) # create filesystem fs.mkfs(self.devname) # set EXT3/4 options if fs.type.startswith('ext'): # max mounts before check (-1 = disable) system2(('/sbin/tune2fs', '-c', '1', self.devname)) # time based (3m = 3 month) system2(('/sbin/tune2fs', '-i', '3m', self.devname)) # set label label = fs.get_label(vol_entry.device) if label: fs.set_label(self.devname, label) LOG.debug('Image %s formatted', self.devname)
def mount(self): self._check(mpoint=True) mounted_to = self._get_device_letter(self.device) if mounted_to != self.mpoint: if not re.match(r'^[a-zA-Z]$', self.mpoint): raise storage2.StorageError( "Mount point must be a single letter. Given: %s" % self.mpoint) LOG.debug('Assigning letter %s to %s', self.mpoint, self.id) self._assign_letter(self.device, self.mpoint) base.bus.fire("block_device_mounted", volume=self) try: getattr(self, 'label') except AttributeError: pass else: fs = storage2.filesystem(self.fstype) if fs.get_label(self.mpoint) != self.label: LOG.debug('Setting label "{}" for device id="{}"'.format( self.label, self.id)) fs.set_label(self.mpoint, self.label) elif self.label: LOG.debug( 'Label for device id="{}" has already been set, skipping.' .format(self.device))
def _format_image(self): LOG.info("Formatting image") vol_entry = [v for v in self._mtab if v.device.startswith('/dev')][0] if vol_entry.device == '/dev/root' and not os.path.exists(vol_entry.device): vol_entry = [v for v in mount.mounts('/etc/mtab') if v.device.startswith('/dev')][0] fs = filesystem(vol_entry.fstype) # create filesystem fs.mkfs(self.devname) # set EXT3/4 options if fs.type.startswith('ext'): # max mounts before check (-1 = disable) system2(('/sbin/tune2fs', '-c', '1', self.devname)) # time based (3m = 3 month) system2(('/sbin/tune2fs', '-i', '3m', self.devname)) # set label label = fs.get_label(vol_entry.device) if label: fs.set_label(self.devname, label) LOG.debug('Image %s formatted', self.devname)
def mkfs(self, force=False): self._check() if not force and self.is_fs_created(): raise storage2.OperationError( 'Filesystem on device %s is already created' % self.device) fs = storage2.filesystem(self.fstype) LOG.info('Creating filesystem on %s', self.device) fs.mkfs(self.device)
def mkfs(self): self._check() if self.fscreated: raise storage2.OperationError( 'fscreated flag is active. Filesystem creation denied ' 'to preserve the original filesystem. If you wish to ' 'proceed anyway set fscreated=False and retry') fs = storage2.filesystem(self.fstype) LOG.info('Creating filesystem on %s', self.device) fs.mkfs(self.device) self.fscreated = True
def mount(self): # Workaround : cindervolume remounts ro sometimes, fsck it first mounted_to = self.mounted_to() if self.is_fs_created() and not mounted_to: self._check_attr('device') self._check_attr('fstype') fs = storage2.filesystem(self.fstype) if fs.type.startswith('ext'): rcode = linux.system(("/sbin/e2fsck", "-fy", self.device), raise_exc=False)[2] if rcode not in (0, 1): raise storage2.StorageError('Fsck failed to correct file system errors') super(CinderVolume, self).mount()
def make_volume(self, config, mpoint, mount_vol=False): config['type'] = 'ebs' LOG.debug('Creating ebs volume') fstype = None for v in mount.mounts('/etc/mtab'): if v.device.startswith('/dev'): fstype = v.fstype break volume = create_volume(config, fstype=filesystem(fstype)) volume.mpoint = mpoint volume.ensure(mount=True, mkfs=True) if not mount_vol: volume.umount() LOG.debug('Volume created %s' % volume.device) return volume
def rebundle(self): rebundle_dir = tempfile.mkdtemp() try: pl = bus.platform proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.new_storage_client() tmp_mount_dir = os.path.join(rebundle_dir, 'root') os.makedirs(tmp_mount_dir) image_name = 'disk.raw' image_path = os.path.join(rebundle_dir, image_name) root_size = coreutils.statvfs('/')['size'] LOG.debug('Creating image file %s' % image_path) with open(image_path, 'w') as f: f.truncate(root_size + 1 * 1024) try: LOG.debug('Creating partition table on image') system(('parted', image_path, 'mklabel', 'msdos')) system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1, str(root_size / (1024 * 1024)))) # Map disk image out = system(('kpartx', '-av', image_path))[0] try: loop = re.search('(/dev/loop\d+)', out).group(1) root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1] LOG.info('Creating filesystem') storage2.filesystem('ext4').mkfs(root_dev_name) dev_uuid = uuid.uuid4() system(('tune2fs', '-U', str(dev_uuid), root_dev_name)) mount.mount(root_dev_name, tmp_mount_dir) try: lines = system(('/bin/mount', '-l'))[0].splitlines() exclude_dirs = set() for line in lines: mpoint = line.split()[2] if mpoint != '/': exclude_dirs.add(mpoint) exclude_dirs.update(self.exclude_dirs) excludes = [ os.path.join(ex, '**') for ex in exclude_dirs ] excludes.extend(self.exclude_files) excludes.extend(self._excludes) LOG.info('Copying root filesystem to image') rsync('/', tmp_mount_dir, archive=True, hard_links=True, times=True, sparse=True, exclude=excludes) LOG.info('Cleanup image') self._create_spec_devices(tmp_mount_dir) LOG.debug('Removing roles-builder user') sh = pexpect.spawn('/bin/sh') try: sh.sendline('chroot %s' % tmp_mount_dir) sh.expect('#') sh.sendline('userdel -rf %s' % ROLEBUILDER_USER) sh.expect('#') finally: sh.close() """ Patch fstab""" fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab') if os.path.exists(fstab_path): with open(fstab_path) as f: fstab = f.read() new_fstab = re.sub('UUID=\S+\s+/\s+(.*)', 'UUID=%s / \\1' % dev_uuid, fstab) with open(fstab_path, 'w') as f: f.write(new_fstab) finally: mount.umount(root_dev_name) finally: system(('kpartx', '-d', image_path)) LOG.info('Compressing image.') arch_name = '%s.tar.gz' % self._role_name.lower() arch_path = os.path.join(rebundle_dir, arch_name) tar = Tar() tar.create().gzip().sparse() tar.archive(arch_path) tar.add(image_name, rebundle_dir) system(str(tar), shell=True) finally: os.unlink(image_path) try: LOG.info('Uploading compressed image to cloud storage') uploader = transfer.Transfer(logger=LOG) tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint( 1, 1000000), int(time.time())) try: remote_path = 'gcs://%s/' % tmp_bucket_name uploader.upload((arch_path, ), remote_path) except: try: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() except: pass cloudstorage.buckets().delete( bucket=tmp_bucket_name).execute() raise finally: os.unlink(arch_path) finally: shutil.rmtree(rebundle_dir) try: goog_image_name = self._role_name.lower().replace('_', '-') LOG.info('Registering new image %s' % goog_image_name) # TODO: check duplicate names compute = pl.new_compute_client() current_image_fq = pl.get_image().split('/') current_img_project = current_image_fq[1] current_img_name = current_image_fq[3] current_img_obj = compute.images().get( project=current_img_project, image=current_img_name).execute() kernel = current_img_obj['preferredKernel'] image_url = 'http://storage.googleapis.com/%s/%s' % ( tmp_bucket_name, arch_name) req_body = dict(name=goog_image_name, sourceType='RAW', preferredKernel=kernel, rawDisk=dict(containerType='TAR', source=image_url)) req = compute.images().insert(project=proj_id, body=req_body) operation = req.execute()['name'] LOG.info('Waiting for image to register') def image_is_ready(): req = compute.globalOperations().get(project=proj_id, operation=operation) res = req.execute() if res['status'] == 'DONE': if res.get('error'): errors = [] for e in res['error']['errors']: err_text = '%s: %s' % (e['code'], e['message']) errors.append(err_text) raise Exception('\n'.join(errors)) return True return False wait_until(image_is_ready, logger=LOG, timeout=600) finally: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() return '%s/images/%s' % (proj_name, goog_image_name)
def _ensure(self): def get_lv_size_kwarg(size): kwd = dict() if '%' in str(size): kwd['extents'] = size else: try: int(size) kwd['size'] = '%sG' % size except: kwd['size'] = size return kwd if self.snap: pvs = [] try: for snap in self.snap['pv_snaps']: snap = storage2.snapshot(snap) vol = storage2.volume(type=snap.type, snap=snap) vol.ensure() pvs.append(vol) except: for pv in pvs: pv.destroy() raise self.pvs = pvs self.vg = self.snap['vg'] self.name = self.snap['name'] pv_volumes = [] for pv_volume in self.pvs: pv_volume = storage2.volume(pv_volume) pv_volume.ensure() pvs = lvm2.pvs() if pv_volume.device not in pvs: pv_volume.umount() lvm2.pvcreate(pv_volume.device) pv_volumes.append(pv_volume) self.pvs = pv_volumes self._check_attr('vg') try: lv_info = self._lvinfo() except lvm2.NotFound: self._check_attr('size') try: lvm2.vgs(self.vg) except lvm2.NotFound: lvm2.vgcreate(self.vg, *[disk.device for disk in self.pvs]) kwds = {'name': self.name} kwds.update(get_lv_size_kwarg(self.size)) lvm2.lvcreate(self.vg, **kwds) lv_info = self._lvinfo() self._config.update({ 'device': lv_info.lv_path, 'snap': None }) pvs_to_extend_vg = [] for pv in self.pvs: pv_info = lvm2.pvs(pv.device)[pv.device] if not pv_info.vg_name: pvs_to_extend_vg.append(pv.device) continue if os.path.basename(self.vg) != pv_info.vg_name: raise storage2.StorageError( 'Can not add physical volume %s to volume group %s: already' ' in volume group %s' % (pv.device, self.vg, pv_info.vg_name)) if pvs_to_extend_vg: lvm2.vgextend(self.vg, *pvs_to_extend_vg) lvm2.lvextend(self.device, **get_lv_size_kwarg(self.size)) if self.is_fs_created(): self.fscreated = True fs = storage2.filesystem(self.fstype) if fs.features.get('resizable'): fs.resize(self.device) if lv_info.lv_attr[4] == '-': lvm2.lvchange(self.device, available='y')
def grow(self, **growth): """ Grow (and/or alternate, e.g.: change ebs type to io1) volume and fs. Method creates clone of current volume, increases it's size and attaches it to the same place. In case of error, old volume attaches back. Old volume detached, but not destroyed. :param growth: Volume type-dependent rules for volume growth :type growth: dict :param resize_fs: Resize fs on device after it's growth or not :type resize_fs: bool :return: New, bigger (or altered) volume instance :rtype: Volume """ if not self.features.get('grow'): raise storage2.StorageError("%s volume type does not'" " support grow." % self.type) # No id, no growth if not self.id: raise storage2.StorageError('Failed to grow volume: ' 'volume has no id.') # Resize_fs is true by default resize_fs = growth.pop('resize_fs', True) self.check_growth(**growth) was_mounted = self.mounted_to() if self.device else False new_vol = None try: LOG.info('Detaching volume %s', self.id) self.detach() new_vol = self.clone() self._grow(new_vol, **growth) if resize_fs: fs_created = new_vol.detect_fstype() if self.fstype: LOG.info('Resizing filesystem') fs = storage2.filesystem(fstype=self.fstype) umount_on_resize = fs.features.get('umount_on_resize') if fs_created: if umount_on_resize: if new_vol.mounted_to(): new_vol.umount() fs.resize(new_vol.device) if was_mounted: new_vol.mount() else: new_vol.mount() fs.resize(new_vol.device) if not was_mounted: new_vol.umount() except: err_type, err_val, trace = sys.exc_info() LOG.warn('Failed to grow volume: %s. Trying to attach old volume', err_val) try: if new_vol: try: new_vol.destroy(force=True, remove_disks=True) except: destr_err = sys.exc_info()[1] LOG.error('Enlarged volume destruction failed: %s' % destr_err) self.ensure(mount=bool(was_mounted)) except: e = sys.exc_info()[1] err_val = str( err_val) + '\nFailed to restore old volume: %s' % e err_val = 'Volume growth failed: %s' % err_val raise storage2.StorageError, err_val, trace return new_vol
def _ensure(self): def get_lv_size_kwarg(size): kwd = dict() if '%' in str(size): kwd['extents'] = size else: try: int(size) kwd['size'] = '%sG' % size except: kwd['size'] = size return kwd if self.snap: pvs = [] try: for snap in self.snap['pv_snaps']: snap = storage2.snapshot(snap) vol = storage2.volume(type=snap.type, snap=snap) vol.ensure() pvs.append(vol) except: for pv in pvs: pv.destroy() raise self.pvs = pvs self.vg = self.snap['vg'] self.name = self.snap['name'] pv_volumes = [] for pv_volume in self.pvs: pv_volume = storage2.volume(pv_volume) pv_volume.ensure() pvs = lvm2.pvs() if pv_volume.device not in pvs: if pv_volume.mounted_to(): pv_volume.umount() lvm2.pvcreate(pv_volume.device) pv_volumes.append(pv_volume) self.pvs = pv_volumes self._check_attr('vg') try: lv_info = self._lvinfo() except lvm2.NotFound: self._check_attr('size') try: lvm2.vgs(self.vg) except lvm2.NotFound: lvm2.vgcreate(self.vg, *[disk.device for disk in self.pvs]) kwds = {'name': self.name} kwds.update(get_lv_size_kwarg(self.size)) lvm2.lvcreate(self.vg, **kwds) lv_info = self._lvinfo() self._config.update({'device': lv_info.lv_path, 'snap': None}) pvs_to_extend_vg = [] for pv in self.pvs: pv_info = lvm2.pvs(pv.device).popitem()[1] if not pv_info.vg_name: pvs_to_extend_vg.append(pv_info.pv_name) continue if os.path.basename(self.vg) != pv_info.vg_name: raise storage2.StorageError( 'Can not add physical volume %s to volume group %s: already' ' in volume group %s' % (pv_info.pv_name, self.vg, pv_info.vg_name)) if pvs_to_extend_vg: lvm2.vgextend(self.vg, *pvs_to_extend_vg) lvm2.lvextend(self.device, **get_lv_size_kwarg(self.size)) if self.is_fs_created(): fs = storage2.filesystem(self.fstype) if fs.features.get('resizable'): fs.resize(self.device) if lv_info.lv_attr[4] == '-': lvm2.lvchange(self.device, available='y') util.wait_until(lambda: os.path.exists(self.device), sleep=1, timeout=30, start_text='Waiting for device %s' % self.device, error_text='Device %s not available' % self.device)
def rebundle(self): rebundle_dir = tempfile.mkdtemp() try: pl = bus.platform proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.new_storage_client() tmp_mount_dir = os.path.join(rebundle_dir, 'root') os.makedirs(tmp_mount_dir) image_name = 'disk.raw' image_path = os.path.join(rebundle_dir, image_name) root_size = coreutils.statvfs('/')['size'] LOG.debug('Creating image file %s' % image_path) with open(image_path, 'w') as f: f.truncate(root_size + 1*1024) try: LOG.debug('Creating partition table on image') system(('parted', image_path, 'mklabel', 'msdos')) system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1, str(root_size/(1024*1024)))) # Map disk image out = system(('kpartx', '-av', image_path))[0] try: loop = re.search('(/dev/loop\d+)', out).group(1) root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1] LOG.info('Creating filesystem') storage2.filesystem('ext4').mkfs(root_dev_name) dev_uuid = uuid.uuid4() system(('tune2fs', '-U', str(dev_uuid), root_dev_name)) mount.mount(root_dev_name, tmp_mount_dir) try: lines = system(('/bin/mount', '-l'))[0].splitlines() exclude_dirs = set() for line in lines: mpoint = line.split()[2] if mpoint != '/': exclude_dirs.add(mpoint) exclude_dirs.update(self.exclude_dirs) excludes = [os.path.join(ex, '**') for ex in exclude_dirs] excludes.extend(self.exclude_files) excludes.extend(self._excludes) LOG.info('Copying root filesystem to image') rsync('/', tmp_mount_dir, archive=True, hard_links=True, times=True, sparse=True, exclude=excludes) LOG.info('Cleanup image') self._create_spec_devices(tmp_mount_dir) LOG.debug('Removing roles-builder user') sh = pexpect.spawn('/bin/sh') try: sh.sendline('chroot %s' % tmp_mount_dir) sh.expect('#') sh.sendline('userdel -rf %s' % ROLEBUILDER_USER) sh.expect('#') finally: sh.close() """ Patch fstab""" fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab') if os.path.exists(fstab_path): with open(fstab_path) as f: fstab = f.read() new_fstab = re.sub('UUID=\S+\s+/\s+(.*)', 'UUID=%s / \\1' % dev_uuid, fstab) with open(fstab_path, 'w') as f: f.write(new_fstab) finally: mount.umount(root_dev_name) finally: system(('kpartx', '-d', image_path)) LOG.info('Compressing image.') arch_name = '%s.tar.gz' % self._role_name.lower() arch_path = os.path.join(rebundle_dir, arch_name) tar = Tar() tar.create().gzip().sparse() tar.archive(arch_path) tar.add(image_name, rebundle_dir) system(str(tar), shell=True) finally: os.unlink(image_path) try: LOG.info('Uploading compressed image to cloud storage') uploader = transfer.Transfer(logger=LOG) tmp_bucket_name = 'scalr-images-%s-%s' % ( random.randint(1,1000000), int(time.time())) try: remote_path = 'gcs://%s/' % tmp_bucket_name uploader.upload((arch_path,), remote_path) except: try: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() except: pass cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() raise finally: os.unlink(arch_path) finally: shutil.rmtree(rebundle_dir) try: goog_image_name = self._role_name.lower().replace('_', '-') LOG.info('Registering new image %s' % goog_image_name) # TODO: check duplicate names compute = pl.new_compute_client() current_image_fq = pl.get_image().split('/') current_img_project = current_image_fq[1] current_img_name = current_image_fq[3] current_img_obj = compute.images().get(project=current_img_project, image=current_img_name).execute() kernel = current_img_obj['preferredKernel'] image_url = 'http://storage.googleapis.com/%s/%s' % ( tmp_bucket_name, arch_name) req_body = dict( name=goog_image_name, sourceType='RAW', preferredKernel=kernel, rawDisk=dict( containerType='TAR', source=image_url ) ) req = compute.images().insert(project=proj_id, body=req_body) operation = req.execute()['name'] LOG.info('Waiting for image to register') def image_is_ready(): req = compute.globalOperations().get(project=proj_id, operation=operation) res = req.execute() if res['status'] == 'DONE': if res.get('error'): errors = [] for e in res['error']['errors']: err_text = '%s: %s' % (e['code'], e['message']) errors.append(err_text) raise Exception('\n'.join(errors)) return True return False wait_until(image_is_ready, logger=LOG, timeout=600) finally: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() return '%s/images/%s' % (proj_name, goog_image_name)
def grow(self, **growth): """ Grow (and/or alternate, e.g.: change ebs type to io1) volume and fs. Method creates clone of current volume, increases it's size and attaches it to the same place. In case of error, old volume attaches back. Old volume detached, but not destroyed. :param growth: Volume type-dependent rules for volume growth :type growth: dict :param resize_fs: Resize fs on device after it's growth or not :type resize_fs: bool :return: New, bigger (or altered) volume instance :rtype: Volume """ if not self.features.get('grow'): raise storage2.StorageError("%s volume type does not'" " support grow." % self.type) # No id, no growth if not self.id: raise storage2.StorageError('Failed to grow volume: ' 'volume has no id.') # Resize_fs is true by default resize_fs = growth.pop('resize_fs', True) self.check_growth(**growth) was_mounted = self.mounted_to() if self.device else False new_vol = None try: LOG.info('Detaching volume %s', self.id) self.detach() new_vol = self.clone() self._grow(new_vol, **growth) if resize_fs: fs_created = new_vol.detect_fstype() if self.fstype: LOG.info('Resizing filesystem') fs = storage2.filesystem(fstype=self.fstype) umount_on_resize = fs.features.get('umount_on_resize') if fs_created: if umount_on_resize: if new_vol.mounted_to(): new_vol.umount() fs.resize(new_vol.device) if was_mounted: new_vol.mount() else: new_vol.mount() fs.resize(new_vol.device) if not was_mounted: new_vol.umount() except: err_type, err_val, trace = sys.exc_info() LOG.warn('Failed to grow volume: %s. Trying to attach old volume', err_val) try: if new_vol: try: new_vol.destroy(force=True, remove_disks=True) except: destr_err = sys.exc_info()[1] LOG.error('Enlarged volume destruction failed: %s' % destr_err) self.ensure(mount=bool(was_mounted)) except: e = sys.exc_info()[1] err_val = str(err_val) + '\nFailed to restore old volume: %s' % e err_val = 'Volume growth failed: %s' % err_val raise storage2.StorageError, err_val, trace return new_vol
def test_create(self, exists, modprobe, pkgmgr): fs = storage2.filesystem('xfs') assert fs.type == 'xfs' modprobe.assert_called_once_with('xfs') pkgmgr.installed.assert_called_with(fs.os_packages[0])