Esempio n. 1
0
def mount_ex(device, 
             mpoint='/mnt',
             options=None,
             make_fs=False,
             fstype='ext3',
             auto_mount=False):
    if not os.path.exists(mpoint):
        os.makedirs(mpoint)
    
    options = options or ('-t', 'auto')
    
    if make_fs:
        from scalarizr import storage2
        storage2.filesystem(fstype).mkfs(device)
    
    out = mount(device, mpoint, *options)[0]
    
    if " ".join(options).find("loop") == -1:
        mtab = mounts()     
        if not mtab.contains(device):
            raise MountError("Cannot mount device '%s'. %s" % (device, out),
                             MountError.CANNOT_MOUNT)
    
    if auto_mount:
        _fstab = fstab()
        if not _fstab.contains(device, mpoint=mpoint, reload=True):
            opts = "defaults"
            if linux.os.ubuntu and linux.os['version'] >= (10, 4):
                opts += ',comment=cloudconfig,nobootwait'
            _fstab.append(device, mpoint, options=opts)
Esempio n. 2
0
def mount_ex(device,
             mpoint='/mnt',
             options=None,
             make_fs=False,
             fstype='ext3',
             auto_mount=False):
    if not os.path.exists(mpoint):
        os.makedirs(mpoint)

    options = options or ('-t', 'auto')

    if make_fs:
        from scalarizr import storage2
        storage2.filesystem(fstype).mkfs(device)

    out = mount(device, mpoint, *options)[0]

    if " ".join(options).find("loop") == -1:
        mtab = mounts()
        if not mtab.contains(device):
            raise MountError("Cannot mount device '%s'. %s" % (device, out),
                             MountError.CANNOT_MOUNT)

    if auto_mount:
        _fstab = fstab()
        if not _fstab.contains(device, mpoint=mpoint, reload=True):
            opts = "defaults"
            if linux.os.ubuntu and linux.os['version'] >= (10, 4):
                opts += ',comment=cloudconfig,nobootwait'
            _fstab.append(device, mpoint, options=opts)
Esempio n. 3
0
    def _format_image(self):
        LOG.info("Formatting image")

        vol_entry = [v for v in self._mtab if v.device.startswith('/dev')][0]
        if vol_entry.device == '/dev/root' and not os.path.exists(
                vol_entry.device):
            vol_entry = [
                v for v in mount.mounts('/etc/mtab')
                if v.device.startswith('/dev')
            ][0]
        fs = filesystem(vol_entry.fstype)

        # create filesystem
        fs.mkfs(self.devname)

        # set EXT3/4 options
        if fs.type.startswith('ext'):
            # max mounts before check (-1 = disable)
            system2(('/sbin/tune2fs', '-c', '1', self.devname))
            # time based (3m = 3 month)
            system2(('/sbin/tune2fs', '-i', '3m', self.devname))

        # set label
        label = fs.get_label(vol_entry.device)
        if label:
            fs.set_label(self.devname, label)

        LOG.debug('Image %s formatted', self.devname)
Esempio n. 4
0
        def mount(self):
            self._check(mpoint=True)
            mounted_to = self._get_device_letter(self.device)
            if mounted_to != self.mpoint:
                if not re.match(r'^[a-zA-Z]$', self.mpoint):
                    raise storage2.StorageError(
                        "Mount point must be a single letter. Given: %s" %
                        self.mpoint)

                LOG.debug('Assigning letter %s to %s', self.mpoint, self.id)
                self._assign_letter(self.device, self.mpoint)
                base.bus.fire("block_device_mounted", volume=self)

            try:
                getattr(self, 'label')
            except AttributeError:
                pass
            else:
                fs = storage2.filesystem(self.fstype)
                if fs.get_label(self.mpoint) != self.label:
                    LOG.debug('Setting label "{}" for device id="{}"'.format(
                        self.label, self.id))
                    fs.set_label(self.mpoint, self.label)
                elif self.label:
                    LOG.debug(
                        'Label for device id="{}" has already been set, skipping.'
                        .format(self.device))
Esempio n. 5
0
    def _format_image(self):
        LOG.info("Formatting image")

        vol_entry = [v for v in self._mtab
                                        if v.device.startswith('/dev')][0]
        if vol_entry.device == '/dev/root' and not os.path.exists(vol_entry.device):
            vol_entry = [v for v in mount.mounts('/etc/mtab')
                            if v.device.startswith('/dev')][0]
        fs = filesystem(vol_entry.fstype)

        # create filesystem
        fs.mkfs(self.devname)

        # set EXT3/4 options
        if fs.type.startswith('ext'):
            # max mounts before check (-1 = disable)
            system2(('/sbin/tune2fs', '-c', '1', self.devname))
            # time based (3m = 3 month)
            system2(('/sbin/tune2fs', '-i', '3m', self.devname))

        # set label
        label = fs.get_label(vol_entry.device)
        if label:
            fs.set_label(self.devname, label)

        LOG.debug('Image %s formatted', self.devname)
Esempio n. 6
0
    def mkfs(self, force=False):
        self._check()
        if not force and self.is_fs_created():
            raise storage2.OperationError(
                                            'Filesystem on device %s is already created' % self.device)

        fs = storage2.filesystem(self.fstype)
        LOG.info('Creating filesystem on %s', self.device)
        fs.mkfs(self.device)
Esempio n. 7
0
    def mkfs(self, force=False):
        self._check()
        if not force and self.is_fs_created():
            raise storage2.OperationError(
                'Filesystem on device %s is already created' % self.device)

        fs = storage2.filesystem(self.fstype)
        LOG.info('Creating filesystem on %s', self.device)
        fs.mkfs(self.device)
Esempio n. 8
0
	def mkfs(self):
		self._check()
		if self.fscreated:
			raise storage2.OperationError(
					'fscreated flag is active. Filesystem creation denied '
					'to preserve the original filesystem. If you wish to '
					'proceed anyway set fscreated=False and retry')
		fs = storage2.filesystem(self.fstype)
		LOG.info('Creating filesystem on %s', self.device)
		fs.mkfs(self.device)
		self.fscreated = True
Esempio n. 9
0
 def mount(self):
     # Workaround : cindervolume remounts ro sometimes, fsck it first
     mounted_to = self.mounted_to()
     if self.is_fs_created() and not mounted_to:
         self._check_attr('device')
         self._check_attr('fstype')
         fs = storage2.filesystem(self.fstype)
         if fs.type.startswith('ext'):
             rcode = linux.system(("/sbin/e2fsck", "-fy", self.device), raise_exc=False)[2]
             if rcode not in (0, 1):
                 raise storage2.StorageError('Fsck failed to correct file system errors')
     super(CinderVolume, self).mount()
Esempio n. 10
0
    def make_volume(self, config, mpoint, mount_vol=False):
        config['type'] = 'ebs'

        LOG.debug('Creating ebs volume')
        fstype = None
        for v in mount.mounts('/etc/mtab'):
            if v.device.startswith('/dev'):
                fstype = v.fstype
                break
        volume = create_volume(config, fstype=filesystem(fstype))
        volume.mpoint = mpoint
        volume.ensure(mount=True, mkfs=True)
        if not mount_vol:
            volume.umount()
        LOG.debug('Volume created %s' % volume.device)
        return volume
Esempio n. 11
0
    def make_volume(self, config, mpoint, mount_vol=False):
        config['type'] = 'ebs'

        LOG.debug('Creating ebs volume')
        fstype = None
        for v in mount.mounts('/etc/mtab'):
            if v.device.startswith('/dev'):
                fstype = v.fstype
                break
        volume = create_volume(config, fstype=filesystem(fstype))
        volume.mpoint = mpoint
        volume.ensure(mount=True, mkfs=True)
        if not mount_vol:
            volume.umount()
        LOG.debug('Volume created %s' % volume.device)
        return volume
Esempio n. 12
0
    def rebundle(self):
        rebundle_dir = tempfile.mkdtemp()

        try:
            pl = bus.platform
            proj_id = pl.get_numeric_project_id()
            proj_name = pl.get_project_id()
            cloudstorage = pl.new_storage_client()

            tmp_mount_dir = os.path.join(rebundle_dir, 'root')
            os.makedirs(tmp_mount_dir)

            image_name = 'disk.raw'
            image_path = os.path.join(rebundle_dir, image_name)

            root_size = coreutils.statvfs('/')['size']
            LOG.debug('Creating image file %s' % image_path)
            with open(image_path, 'w') as f:
                f.truncate(root_size + 1 * 1024)

            try:

                LOG.debug('Creating partition table on image')
                system(('parted', image_path, 'mklabel', 'msdos'))
                system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1,
                        str(root_size / (1024 * 1024))))

                # Map disk image
                out = system(('kpartx', '-av', image_path))[0]
                try:
                    loop = re.search('(/dev/loop\d+)', out).group(1)
                    root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1]

                    LOG.info('Creating filesystem')
                    storage2.filesystem('ext4').mkfs(root_dev_name)
                    dev_uuid = uuid.uuid4()
                    system(('tune2fs', '-U', str(dev_uuid), root_dev_name))

                    mount.mount(root_dev_name, tmp_mount_dir)
                    try:
                        lines = system(('/bin/mount', '-l'))[0].splitlines()
                        exclude_dirs = set()
                        for line in lines:
                            mpoint = line.split()[2]
                            if mpoint != '/':
                                exclude_dirs.add(mpoint)

                        exclude_dirs.update(self.exclude_dirs)

                        excludes = [
                            os.path.join(ex, '**') for ex in exclude_dirs
                        ]
                        excludes.extend(self.exclude_files)
                        excludes.extend(self._excludes)

                        LOG.info('Copying root filesystem to image')
                        rsync('/',
                              tmp_mount_dir,
                              archive=True,
                              hard_links=True,
                              times=True,
                              sparse=True,
                              exclude=excludes)

                        LOG.info('Cleanup image')
                        self._create_spec_devices(tmp_mount_dir)

                        LOG.debug('Removing roles-builder user')
                        sh = pexpect.spawn('/bin/sh')
                        try:
                            sh.sendline('chroot %s' % tmp_mount_dir)
                            sh.expect('#')
                            sh.sendline('userdel -rf %s' % ROLEBUILDER_USER)
                            sh.expect('#')
                        finally:
                            sh.close()
                        """ Patch fstab"""
                        fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab')
                        if os.path.exists(fstab_path):
                            with open(fstab_path) as f:
                                fstab = f.read()

                            new_fstab = re.sub('UUID=\S+\s+/\s+(.*)',
                                               'UUID=%s / \\1' % dev_uuid,
                                               fstab)

                            with open(fstab_path, 'w') as f:
                                f.write(new_fstab)

                    finally:
                        mount.umount(root_dev_name)
                finally:
                    system(('kpartx', '-d', image_path))

                LOG.info('Compressing image.')
                arch_name = '%s.tar.gz' % self._role_name.lower()
                arch_path = os.path.join(rebundle_dir, arch_name)

                tar = Tar()
                tar.create().gzip().sparse()
                tar.archive(arch_path)
                tar.add(image_name, rebundle_dir)
                system(str(tar), shell=True)

            finally:
                os.unlink(image_path)

            try:
                LOG.info('Uploading compressed image to cloud storage')
                uploader = transfer.Transfer(logger=LOG)
                tmp_bucket_name = 'scalr-images-%s-%s' % (random.randint(
                    1, 1000000), int(time.time()))

                try:
                    remote_path = 'gcs://%s/' % tmp_bucket_name
                    uploader.upload((arch_path, ), remote_path)
                except:
                    try:
                        objs = cloudstorage.objects()
                        objs.delete(bucket=tmp_bucket_name,
                                    object=arch_name).execute()
                    except:
                        pass

                    cloudstorage.buckets().delete(
                        bucket=tmp_bucket_name).execute()
                    raise

            finally:
                os.unlink(arch_path)

        finally:
            shutil.rmtree(rebundle_dir)

        try:
            goog_image_name = self._role_name.lower().replace('_', '-')
            LOG.info('Registering new image %s' % goog_image_name)
            # TODO: check duplicate names
            compute = pl.new_compute_client()

            current_image_fq = pl.get_image().split('/')
            current_img_project = current_image_fq[1]
            current_img_name = current_image_fq[3]
            current_img_obj = compute.images().get(
                project=current_img_project, image=current_img_name).execute()
            kernel = current_img_obj['preferredKernel']

            image_url = 'http://storage.googleapis.com/%s/%s' % (
                tmp_bucket_name, arch_name)

            req_body = dict(name=goog_image_name,
                            sourceType='RAW',
                            preferredKernel=kernel,
                            rawDisk=dict(containerType='TAR',
                                         source=image_url))

            req = compute.images().insert(project=proj_id, body=req_body)
            operation = req.execute()['name']

            LOG.info('Waiting for image to register')

            def image_is_ready():
                req = compute.globalOperations().get(project=proj_id,
                                                     operation=operation)
                res = req.execute()
                if res['status'] == 'DONE':
                    if res.get('error'):
                        errors = []
                        for e in res['error']['errors']:
                            err_text = '%s: %s' % (e['code'], e['message'])
                            errors.append(err_text)
                        raise Exception('\n'.join(errors))
                    return True
                return False

            wait_until(image_is_ready, logger=LOG, timeout=600)

        finally:
            objs = cloudstorage.objects()
            objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
            cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()

        return '%s/images/%s' % (proj_name, goog_image_name)
Esempio n. 13
0
	def _ensure(self):
		def get_lv_size_kwarg(size):
			kwd = dict()
			if '%' in str(size):
				kwd['extents'] = size
			else:
				try:
					int(size)
					kwd['size'] = '%sG' % size
				except:
					kwd['size'] = size
			return kwd

		if self.snap:
			pvs = []
			try:
				for snap in self.snap['pv_snaps']:
					snap = storage2.snapshot(snap)
					vol = storage2.volume(type=snap.type, snap=snap)
					vol.ensure()
					pvs.append(vol)
			except:
				for pv in pvs:
					pv.destroy()
				raise
			self.pvs = pvs
			self.vg = self.snap['vg']
			self.name = self.snap['name']
		
		pv_volumes = []
		for pv_volume in self.pvs:
			pv_volume = storage2.volume(pv_volume)
			pv_volume.ensure()

			pvs = lvm2.pvs()
			if pv_volume.device not in pvs:
				pv_volume.umount()
				lvm2.pvcreate(pv_volume.device)
			pv_volumes.append(pv_volume)
		self.pvs = pv_volumes

		self._check_attr('vg')
		try:
			lv_info = self._lvinfo()
		except lvm2.NotFound:
			self._check_attr('size')
			
			try:
				lvm2.vgs(self.vg)
			except lvm2.NotFound:
				lvm2.vgcreate(self.vg, *[disk.device for disk in self.pvs])

			kwds = {'name': self.name}
			kwds.update(get_lv_size_kwarg(self.size))

			lvm2.lvcreate(self.vg, **kwds)
			lv_info = self._lvinfo()

		self._config.update({
			'device': lv_info.lv_path,
			'snap': None
		})

		pvs_to_extend_vg = []
		for pv in self.pvs:
			pv_info = lvm2.pvs(pv.device)[pv.device]

			if not pv_info.vg_name:
				pvs_to_extend_vg.append(pv.device)
				continue

			if os.path.basename(self.vg) != pv_info.vg_name:
				raise storage2.StorageError(
					'Can not add physical volume %s to volume group %s: already'
					' in volume group %s' %
					(pv.device, self.vg, pv_info.vg_name))

		if pvs_to_extend_vg:
			lvm2.vgextend(self.vg, *pvs_to_extend_vg)
			lvm2.lvextend(self.device, **get_lv_size_kwarg(self.size))
			if self.is_fs_created():
				self.fscreated = True
				fs = storage2.filesystem(self.fstype)
				if fs.features.get('resizable'):
					fs.resize(self.device)

		if lv_info.lv_attr[4] == '-':
			lvm2.lvchange(self.device, available='y')
Esempio n. 14
0
    def grow(self, **growth):
        """
        Grow (and/or alternate, e.g.: change ebs type to io1) volume and fs.
        Method creates clone of current volume, increases it's size and
        attaches it to the same place. In case of error, old volume attaches back.

        Old volume detached, but not destroyed.

        :param growth: Volume type-dependent rules for volume growth
        :type growth: dict
        :param resize_fs: Resize fs on device after it's growth or not
        :type resize_fs: bool
        :return: New, bigger (or altered) volume instance
        :rtype: Volume
        """

        if not self.features.get('grow'):
            raise storage2.StorageError("%s volume type does not'"
                                        " support grow." % self.type)

        # No id, no growth
        if not self.id:
            raise storage2.StorageError('Failed to grow volume: '
                                        'volume has no id.')

        # Resize_fs is true by default
        resize_fs = growth.pop('resize_fs', True)

        self.check_growth(**growth)
        was_mounted = self.mounted_to() if self.device else False

        new_vol = None
        try:
            LOG.info('Detaching volume %s', self.id)
            self.detach()
            new_vol = self.clone()
            self._grow(new_vol, **growth)
            if resize_fs:
                fs_created = new_vol.detect_fstype()

                if self.fstype:
                    LOG.info('Resizing filesystem')
                    fs = storage2.filesystem(fstype=self.fstype)
                    umount_on_resize = fs.features.get('umount_on_resize')

                    if fs_created:
                        if umount_on_resize:
                            if new_vol.mounted_to():
                                new_vol.umount()
                            fs.resize(new_vol.device)
                            if was_mounted:
                                new_vol.mount()
                        else:
                            new_vol.mount()
                            fs.resize(new_vol.device)
                            if not was_mounted:
                                new_vol.umount()

        except:
            err_type, err_val, trace = sys.exc_info()
            LOG.warn('Failed to grow volume: %s. Trying to attach old volume',
                     err_val)
            try:
                if new_vol:
                    try:
                        new_vol.destroy(force=True, remove_disks=True)
                    except:
                        destr_err = sys.exc_info()[1]
                        LOG.error('Enlarged volume destruction failed: %s' %
                                  destr_err)

                self.ensure(mount=bool(was_mounted))
            except:
                e = sys.exc_info()[1]
                err_val = str(
                    err_val) + '\nFailed to restore old volume: %s' % e

            err_val = 'Volume growth failed: %s' % err_val
            raise storage2.StorageError, err_val, trace

        return new_vol
Esempio n. 15
0
    def _ensure(self):
        def get_lv_size_kwarg(size):
            kwd = dict()
            if '%' in str(size):
                kwd['extents'] = size
            else:
                try:
                    int(size)
                    kwd['size'] = '%sG' % size
                except:
                    kwd['size'] = size
            return kwd

        if self.snap:
            pvs = []
            try:
                for snap in self.snap['pv_snaps']:
                    snap = storage2.snapshot(snap)
                    vol = storage2.volume(type=snap.type, snap=snap)
                    vol.ensure()
                    pvs.append(vol)
            except:
                for pv in pvs:
                    pv.destroy()
                raise
            self.pvs = pvs
            self.vg = self.snap['vg']
            self.name = self.snap['name']

        pv_volumes = []
        for pv_volume in self.pvs:
            pv_volume = storage2.volume(pv_volume)
            pv_volume.ensure()

            pvs = lvm2.pvs()
            if pv_volume.device not in pvs:
                if pv_volume.mounted_to():
                    pv_volume.umount()
                lvm2.pvcreate(pv_volume.device)
            pv_volumes.append(pv_volume)
        self.pvs = pv_volumes

        self._check_attr('vg')
        try:
            lv_info = self._lvinfo()
        except lvm2.NotFound:
            self._check_attr('size')

            try:
                lvm2.vgs(self.vg)
            except lvm2.NotFound:
                lvm2.vgcreate(self.vg, *[disk.device for disk in self.pvs])

            kwds = {'name': self.name}
            kwds.update(get_lv_size_kwarg(self.size))

            lvm2.lvcreate(self.vg, **kwds)
            lv_info = self._lvinfo()

        self._config.update({'device': lv_info.lv_path, 'snap': None})

        pvs_to_extend_vg = []
        for pv in self.pvs:
            pv_info = lvm2.pvs(pv.device).popitem()[1]

            if not pv_info.vg_name:
                pvs_to_extend_vg.append(pv_info.pv_name)
                continue

            if os.path.basename(self.vg) != pv_info.vg_name:
                raise storage2.StorageError(
                    'Can not add physical volume %s to volume group %s: already'
                    ' in volume group %s' %
                    (pv_info.pv_name, self.vg, pv_info.vg_name))

        if pvs_to_extend_vg:
            lvm2.vgextend(self.vg, *pvs_to_extend_vg)
            lvm2.lvextend(self.device, **get_lv_size_kwarg(self.size))
            if self.is_fs_created():
                fs = storage2.filesystem(self.fstype)
                if fs.features.get('resizable'):
                    fs.resize(self.device)

        if lv_info.lv_attr[4] == '-':
            lvm2.lvchange(self.device, available='y')
            util.wait_until(lambda: os.path.exists(self.device),
                            sleep=1,
                            timeout=30,
                            start_text='Waiting for device %s' % self.device,
                            error_text='Device %s not available' % self.device)
Esempio n. 16
0
    def rebundle(self):
        rebundle_dir = tempfile.mkdtemp()

        try:
            pl = bus.platform
            proj_id = pl.get_numeric_project_id()
            proj_name = pl.get_project_id()
            cloudstorage = pl.new_storage_client()

            tmp_mount_dir = os.path.join(rebundle_dir, 'root')
            os.makedirs(tmp_mount_dir)

            image_name      = 'disk.raw'
            image_path      = os.path.join(rebundle_dir, image_name)

            root_size = coreutils.statvfs('/')['size']
            LOG.debug('Creating image file %s' % image_path)
            with open(image_path, 'w') as f:
                f.truncate(root_size + 1*1024)

            try:

                LOG.debug('Creating partition table on image')
                system(('parted', image_path, 'mklabel', 'msdos'))
                system(('parted', image_path, 'mkpart', 'primary', 'ext2', 1, str(root_size/(1024*1024))))

                # Map disk image
                out = system(('kpartx', '-av', image_path))[0]
                try:
                    loop = re.search('(/dev/loop\d+)', out).group(1)
                    root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1]

                    LOG.info('Creating filesystem')
                    storage2.filesystem('ext4').mkfs(root_dev_name)
                    dev_uuid = uuid.uuid4()
                    system(('tune2fs', '-U', str(dev_uuid), root_dev_name))

                    mount.mount(root_dev_name, tmp_mount_dir)
                    try:
                        lines = system(('/bin/mount', '-l'))[0].splitlines()
                        exclude_dirs = set()
                        for line in lines:
                            mpoint = line.split()[2]
                            if mpoint != '/':
                                exclude_dirs.add(mpoint)

                        exclude_dirs.update(self.exclude_dirs)

                        excludes = [os.path.join(ex, '**') for ex in exclude_dirs]
                        excludes.extend(self.exclude_files)
                        excludes.extend(self._excludes)

                        LOG.info('Copying root filesystem to image')
                        rsync('/', tmp_mount_dir, archive=True,
                                                                          hard_links=True,
                                                                          times=True,
                                                                          sparse=True,
                                                                          exclude=excludes)

                        LOG.info('Cleanup image')
                        self._create_spec_devices(tmp_mount_dir)

                        LOG.debug('Removing roles-builder user')
                        sh = pexpect.spawn('/bin/sh')
                        try:
                            sh.sendline('chroot %s' % tmp_mount_dir)
                            sh.expect('#')
                            sh.sendline('userdel -rf %s' % ROLEBUILDER_USER)
                            sh.expect('#')
                        finally:
                            sh.close()

                        """ Patch fstab"""
                        fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab')
                        if os.path.exists(fstab_path):
                            with open(fstab_path) as f:
                                fstab = f.read()

                            new_fstab = re.sub('UUID=\S+\s+/\s+(.*)', 'UUID=%s / \\1' % dev_uuid, fstab)

                            with open(fstab_path, 'w') as f:
                                f.write(new_fstab)

                    finally:
                        mount.umount(root_dev_name)
                finally:
                    system(('kpartx', '-d', image_path))

                LOG.info('Compressing image.')
                arch_name = '%s.tar.gz' % self._role_name.lower()
                arch_path = os.path.join(rebundle_dir, arch_name)

                tar = Tar()
                tar.create().gzip().sparse()
                tar.archive(arch_path)
                tar.add(image_name, rebundle_dir)
                system(str(tar), shell=True)

            finally:
                os.unlink(image_path)

            try:
                LOG.info('Uploading compressed image to cloud storage')
                uploader = transfer.Transfer(logger=LOG)
                tmp_bucket_name = 'scalr-images-%s-%s' % (
                                                        random.randint(1,1000000), int(time.time()))

                try:
                    remote_path = 'gcs://%s/' % tmp_bucket_name
                    uploader.upload((arch_path,), remote_path)
                except:
                    try:
                        objs = cloudstorage.objects()
                        objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
                    except:
                        pass

                    cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()
                    raise

            finally:
                os.unlink(arch_path)

        finally:
            shutil.rmtree(rebundle_dir)

        try:
            goog_image_name = self._role_name.lower().replace('_', '-')
            LOG.info('Registering new image %s' % goog_image_name)
            # TODO: check duplicate names
            compute = pl.new_compute_client()

            current_image_fq = pl.get_image().split('/')
            current_img_project = current_image_fq[1]
            current_img_name = current_image_fq[3]
            current_img_obj = compute.images().get(project=current_img_project,
                                                            image=current_img_name).execute()
            kernel = current_img_obj['preferredKernel']

            image_url = 'http://storage.googleapis.com/%s/%s' % (
                                                                            tmp_bucket_name, arch_name)

            req_body = dict(
                    name=goog_image_name,
                    sourceType='RAW',
                    preferredKernel=kernel,
                    rawDisk=dict(
                            containerType='TAR',
                            source=image_url
                    )
            )

            req = compute.images().insert(project=proj_id, body=req_body)
            operation = req.execute()['name']

            LOG.info('Waiting for image to register')
            def image_is_ready():
                req = compute.globalOperations().get(project=proj_id, operation=operation)
                res = req.execute()
                if res['status'] == 'DONE':
                    if res.get('error'):
                        errors = []
                        for e in res['error']['errors']:
                            err_text = '%s: %s' % (e['code'], e['message'])
                            errors.append(err_text)
                        raise Exception('\n'.join(errors))
                    return True
                return False
            wait_until(image_is_ready, logger=LOG, timeout=600)

        finally:
            objs = cloudstorage.objects()
            objs.delete(bucket=tmp_bucket_name, object=arch_name).execute()
            cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute()

        return '%s/images/%s' % (proj_name, goog_image_name)
Esempio n. 17
0
    def grow(self, **growth):
        """
        Grow (and/or alternate, e.g.: change ebs type to io1) volume and fs.
        Method creates clone of current volume, increases it's size and
        attaches it to the same place. In case of error, old volume attaches back.

        Old volume detached, but not destroyed.

        :param growth: Volume type-dependent rules for volume growth
        :type growth: dict
        :param resize_fs: Resize fs on device after it's growth or not
        :type resize_fs: bool
        :return: New, bigger (or altered) volume instance
        :rtype: Volume
        """

        if not self.features.get('grow'):
            raise storage2.StorageError("%s volume type does not'"
                                                                    " support grow." % self.type)

        # No id, no growth
        if not self.id:
            raise storage2.StorageError('Failed to grow volume: '
                                                            'volume has no id.')

        # Resize_fs is true by default
        resize_fs = growth.pop('resize_fs', True)

        self.check_growth(**growth)
        was_mounted = self.mounted_to() if self.device else False

        new_vol = None
        try:
            LOG.info('Detaching volume %s', self.id)
            self.detach()
            new_vol = self.clone()
            self._grow(new_vol, **growth)
            if resize_fs:
                fs_created = new_vol.detect_fstype()

                if self.fstype:
                    LOG.info('Resizing filesystem')
                    fs = storage2.filesystem(fstype=self.fstype)
                    umount_on_resize = fs.features.get('umount_on_resize')

                    if fs_created:
                        if umount_on_resize:
                            if new_vol.mounted_to():
                                new_vol.umount()
                            fs.resize(new_vol.device)
                            if was_mounted:
                                new_vol.mount()
                        else:
                            new_vol.mount()
                            fs.resize(new_vol.device)
                            if not was_mounted:
                                new_vol.umount()

        except:
            err_type, err_val, trace = sys.exc_info()
            LOG.warn('Failed to grow volume: %s. Trying to attach old volume', err_val)
            try:
                if new_vol:
                    try:
                        new_vol.destroy(force=True, remove_disks=True)
                    except:
                        destr_err = sys.exc_info()[1]
                        LOG.error('Enlarged volume destruction failed: %s' % destr_err)

                self.ensure(mount=bool(was_mounted))
            except:
                e = sys.exc_info()[1]
                err_val = str(err_val) + '\nFailed to restore old volume: %s' % e

            err_val = 'Volume growth failed: %s' % err_val
            raise storage2.StorageError, err_val, trace

        return new_vol
Esempio n. 18
0
 def test_create(self, exists, modprobe, pkgmgr):
     fs = storage2.filesystem('xfs')
     assert fs.type == 'xfs'
     modprobe.assert_called_once_with('xfs')
     pkgmgr.installed.assert_called_with(fs.os_packages[0])