def _snapshot(self, description, tags, **kwds): lvm_snap = self._lvm_volume.lvm_snapshot(size='100%FREE') try: snap = storage2.snapshot(type='eph') snap.path = os.path.join(os.path.join( self.cloudfs_dir, snap.id + '.manifest.ini')) lvm_snap_vol = storage2.volume( device=lvm_snap.device, mpoint=tempfile.mkdtemp()) lvm_snap_vol.ensure(mount=True) df_info = filetool.df() df = filter(lambda x: x.mpoint == lvm_snap_vol.mpoint, df_info) snap.size = df[0].used try: transfer = cloudfs.LargeTransfer( src=lvm_snap_vol.mpoint + '/', dst=snap.path, tar_it=True, gzip_it=True, tags=tags) transfer.run() finally: lvm_snap_vol.umount() os.rmdir(lvm_snap_vol.mpoint) finally: lvm_snap.destroy() return snap
def _ensure(self): # snap should be applied after layout: download and extract data. # this could be done on already ensured volume. # Example: resync slave data if not self._lvm_volume: if isinstance(self.disk, basestring) and \ self.disk.startswith('/dev/sd'): self.disk = storage2.volume( type='ec2_ephemeral', name='ephemeral0') self._lvm_volume = storage2.volume( type='lvm', pvs=[self.disk], size=self.size + 'VG', vg=self.vg, name='data') self._lvm_volume.ensure() self.device = self._lvm_volume.device if self.snap: self.snap = storage2.snapshot(self.snap) self.mkfs() tmp_mpoint = not self.mpoint if tmp_mpoint: tmp_mpoint = tempfile.mkdtemp() self.mpoint = tmp_mpoint transfer = cloudfs.LargeTransfer(self.snap.path, self.mpoint + '/') try: self.mount() if hasattr(self.snap, 'size'): df_info = filetool.df() df = filter(lambda x: x.mpoint == self.mpoint, df_info)[0] if df.free < self.snap.size: raise storage2.StorageError('Not enough free space' ' on device %s to restore snapshot.' % self.device) transfer.run() except: e = sys.exc_info()[1] raise storage2.StorageError("Snapshot restore error: %s" % e) finally: try: self.umount() finally: if tmp_mpoint: self.mpoint = None os.rmdir(tmp_mpoint) self.snap = None
def rebundle(self): rebundle_dir = tempfile.mkdtemp() try: pl = bus.platform proj_id = pl.get_numeric_project_id() proj_name = pl.get_project_id() cloudstorage = pl.new_storage_client() tmp_mount_dir = os.path.join(rebundle_dir, 'root') os.makedirs(tmp_mount_dir) image_name = 'disk.raw' image_path = os.path.join(rebundle_dir, image_name) root = filter(lambda x: x.mpoint == '/', filetool.df())[0] LOG.debug('Creating image file %s' % image_path) with open(image_path, 'w') as f: f.truncate(root.size*1024 + 1*1024) try: LOG.debug('Creating partition table on image') system2(('parted', image_path, 'mklabel', 'msdos')) system2(('parted', image_path, 'mkpart', 'primary', 'ext2', 1, str(root.size/1024))) # Map disk image out = system2(('kpartx', '-av', image_path))[0] try: loop = re.search('(/dev/loop\d+)', out).group(1) root_dev_name = '/dev/mapper/%sp1' % loop.split('/')[-1] LOG.info('Creating filesystem') fstool.mkfs(root_dev_name, 'ext4') dev_uuid = uuid.uuid4() system2(('tune2fs', '-U', str(dev_uuid), root_dev_name)) fstool.mount(root_dev_name, tmp_mount_dir) try: lines = system2(('/bin/mount', '-l'))[0].splitlines() exclude_dirs = set() for line in lines: mpoint = line.split()[2] if mpoint != '/': exclude_dirs.add(mpoint) exclude_dirs.update(self.exclude_dirs) rsync = filetool.Rsync() rsync.source('/').dest(tmp_mount_dir).sparse() rsync.hardlinks().archive().times() rsync.exclude([os.path.join(ex, '**') for ex in exclude_dirs]) rsync.exclude(self.exclude_files) rsync.exclude(self._excludes) LOG.info('Copying root filesystem to image') rsync.execute() LOG.info('Cleanup image') self._create_spec_devices(tmp_mount_dir) LOG.debug('Removing roles-builder user') sh = pexpect.spawn('/bin/sh') try: sh.sendline('chroot %s' % tmp_mount_dir) sh.expect('#') sh.sendline('userdel -rf %s' % ROLEBUILDER_USER) sh.expect('#') finally: sh.close() """ Patch fstab""" fstab_path = os.path.join(tmp_mount_dir, 'etc/fstab') if os.path.exists(fstab_path): with open(fstab_path) as f: fstab = f.read() new_fstab = re.sub('UUID=\S+\s+/\s+(.*)', 'UUID=%s / \\1' % dev_uuid, fstab) with open(fstab_path, 'w') as f: f.write(new_fstab) finally: fstool.umount(device=root_dev_name) finally: system2(('kpartx', '-d', image_path)) LOG.info('Compressing image.') arch_name = '%s.tar.gz' % self._role_name.lower() arch_path = os.path.join(rebundle_dir, arch_name) tar = filetool.Tar() tar.create().gzip().sparse() tar.archive(arch_path) tar.add(image_name, rebundle_dir) system2(str(tar), shell=True) finally: os.unlink(image_path) try: LOG.info('Uploading compressed image to cloud storage') uploader = transfer.Transfer(logger=LOG) tmp_bucket_name = 'scalr-images-%s-%s' % ( random.randint(1,1000000), int(time.time())) try: remote_path = 'gcs://%s/' % tmp_bucket_name uploader.upload((arch_path,), remote_path) except: try: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() except: pass cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() raise finally: os.unlink(arch_path) finally: shutil.rmtree(rebundle_dir) try: goog_image_name = self._role_name.lower().replace('_', '-') LOG.info('Registering new image %s' % goog_image_name) # TODO: check duplicate names compute = pl.new_compute_client() image_url = 'http://storage.googleapis.com/%s/%s' % ( tmp_bucket_name, arch_name) req_body = dict( name=goog_image_name, sourceType='RAW', rawDisk=dict( containerType='TAR', source=image_url ) ) req = compute.images().insert(project=proj_id, body=req_body) operation = req.execute()['name'] LOG.info('Waiting for image to register') def image_is_ready(): req = compute.operations().get(project=proj_id, operation=operation) res = req.execute() if res['status'] == 'DONE': if res.get('error'): errors = [] for e in res['error']['errors']: err_text = '%s: %s' % (e['code'], e['message']) errors.append(err_text) raise Exception('\n'.join(errors)) return True return False wait_until(image_is_ready, logger=LOG, timeout=600) finally: objs = cloudstorage.objects() objs.delete(bucket=tmp_bucket_name, object=arch_name).execute() cloudstorage.buckets().delete(bucket=tmp_bucket_name).execute() return '%s/images/%s' % (proj_name, goog_image_name)