Esempio n. 1
0
    def create_snapshot(self, volume_id, snapshot_id, timestamp=None,
                        type_='backup', clone_id=None):
        snapshot = self._get_snapshot(volume_id)
        if snapshot:
            if type_ == 'backup':
                if snapshot['id'] == snapshot_id:
                    return snapshot
            elif type_ == 'clone':
                if snapshot.get('clone_id') == clone_id:
                    return snapshot
            raise AlreadyExists(
                "Volume %s already has a snapshot." % volume_id)

        origin = self.get(volume_id)
        # TODO: support size as kwarg or % of origin.size?
        sizestr = '%sB' % self._max_snapshot_size(origin['size'])

        if type_ == 'backup':
            # TODO: should we prevent create snapshot if timestamp is too old?
            timestamp = int(timestamp or time())
            tag = encode_tag(backup_id=snapshot_id, timestamp=timestamp)
        elif type_ == 'clone':
            tag = encode_tag(clone_id=clone_id)
        else:
            raise ValueError("Invalid snapshot type: %s" % type_)

        try:
            # Create an lvm snapshot
            execute('lvcreate', origin['path'], name=snapshot_id,
                    size=sizestr, snapshot=None, addtag=tag)
            return self.get(snapshot_id)
        except ProcessError, e:
            if e.errcode != 5 or 'already exists' not in e.err:
                raise
            raise AlreadyExists("snapshot id '%s' already in use" % id)
Esempio n. 2
0
    def create_snapshot(self, volume_id, snapshot_id, timestamp=None,
                        type_='backup', clone_id=None):
        snapshot = self._get_snapshot(volume_id)
        if snapshot:
            if type_ == 'backup':
                if snapshot['id'] == snapshot_id:
                    return snapshot
            elif type_ == 'clone':
                if snapshot.get('clone_id') == clone_id:
                    return snapshot
            raise AlreadyExists(
                "Volume %s already has a snapshot." % volume_id)

        origin = self.get(volume_id)
        # TODO: support size as kwarg or % of origin.size?
        sizestr = '%sB' % self._max_snapshot_size(origin['size'])

        if type_ == 'backup':
            # TODO: should we prevent create snapshot if timestamp is too old?
            timestamp = int(timestamp or time())
            tag = encode_tag(backup_id=snapshot_id, timestamp=timestamp)
        elif type_ == 'clone':
            tag = encode_tag(clone_id=clone_id)
        else:
            raise ValueError("Invalid snapshot type: %s" % type_)

        try:
            # Create an lvm snapshot
            execute('lvcreate', origin['path'], name=snapshot_id,
                    size=sizestr, snapshot=None, addtag=tag)
            return self.get(snapshot_id)
        except ProcessError, e:
            if e.errcode != 5 or 'already exists' not in e.err:
                raise
            raise AlreadyExists("snapshot id '%s' already in use" % id)
Esempio n. 3
0
    def clone(self, id=None, src=None, backup=None, size=None):
        """
        This runs a clone job outside of the storage api,
        which is useful for performance testing backup restores
        (Example: storage tools clone volume-clone
          --backup volume-backup --src volume-original)
        """
        # Set basic Logging
        logging.basicConfig()
        # Get the lunr logger
        log = logger.get_logger()
        # Output Debug level info
        log.logger.setLevel(logging.DEBUG)
        # Load the local storage configuration
        conf = LunrConfig.from_storage_conf()
        # Init the volume helper
        volume = VolumeHelper(conf)

        # Attempt to figure out the original volume size
        size = size or str(volume.get(src)['size'] / 1073741824)
        # Size is in gigs
        if not re.match('G', size):
            size = size + 'G'
        # Create a tag to apply to the lvm volume
        tag = encode_tag(source_volume_id=src, backup_id=backup)
        # Create the volume
        execute('lvcreate', volume.volume_group,
                name=id, size=size, addtag=tag)
        # Get info for the newly created volume
        new = volume.get(id)

        with self.timeit():
            print("Starting Backup")
            # Restore volume from the backup
            volume.clone(new, src, backup)
Esempio n. 4
0
File: iscsi.py Progetto: audip/lunr
 def disconnect(self):
     """ logout or close the iscsi connection """
     try:
         execute('iscsiadm', mode='node', targetname=self.iqn,
                 portal=self.ip, logout=None)
     except ProcessError, e:
         logger.exception("iscsi logout failed with '%s'" % e)
         raise ISCSILogoutFailed()
Esempio n. 5
0
 def reparent_vhd_chain(self, chain):
     chain = list(chain)
     while len(chain) > 1:
         child = chain.pop(0)
         parent = chain[0]
         # Because someone hates us, these vhds don't actually point to
         # their real parent file name.
         execute('vhd-util', 'modify', '-n', child, '-p', parent,
                 sudo=False)
Esempio n. 6
0
 def reparent_vhd_chain(self, chain):
     chain = list(chain)
     while len(chain) > 1:
         child = chain.pop(0)
         parent = chain[0]
         # Because someone hates us, these vhds don't actually point to
         # their real parent file name.
         execute('vhd-util', 'modify', '-n', child, '-p', parent,
                 sudo=False)
Esempio n. 7
0
 def cleanup_tmp_vol(self, tmp_vol, convert_dir, scrub_callback):
     if not tmp_vol:
         raise ValueError("No tmp_vol")
     if not os.path.exists(tmp_vol['path']):
         raise ValueError("tmp_vol doesn't exist")
     if convert_dir:
         execute('umount', convert_dir)
         rmtree(convert_dir)
     spawn(NullResource(), self.remove_lvm_volume, tmp_vol,
           callback=scrub_callback, skip_fork=self.skip_fork)
Esempio n. 8
0
 def cleanup_tmp_vol(self, tmp_vol, convert_dir, scrub_callback):
     if not tmp_vol:
         raise ValueError("No tmp_vol")
     if not os.path.exists(tmp_vol['path']):
         raise ValueError("tmp_vol doesn't exist")
     if convert_dir:
         execute('umount', convert_dir)
         rmtree(convert_dir)
     spawn(NullResource(), self.remove_lvm_volume, tmp_vol,
           callback=scrub_callback, skip_fork=self.skip_fork)
Esempio n. 9
0
 def disconnect(self):
     """ logout or close the iscsi connection """
     try:
         execute('iscsiadm',
                 mode='node',
                 targetname=self.iqn,
                 portal=self.ip,
                 logout=None)
     except ProcessError, e:
         logger.exception("iscsi logout failed with '%s'" % e)
         raise ISCSILogoutFailed()
Esempio n. 10
0
File: iscsi.py Progetto: audip/lunr
 def connect(self):
     """ login or open an iscsi connection """
     try:
         portal = "%s:%s" % (self.ip, self.port)
         execute('iscsiadm', mode='discovery', type='sendtargets',
                 portal=self.ip)
         execute('iscsiadm', mode='node', targetname=self.iqn,
                 portal=self.ip, login=None)
     except ProcessError, e:
         logger.exception("iscsi login failed with '%s'" % e)
         raise ISCSILoginFailed()
Esempio n. 11
0
 def rename(self, src, dest):
     """ Rename 'src' to 'dest' """
     try:
         helper = self.load_conf(self.config)
         execute('lvrename', helper.volumes.volume_group, src, dest)
         print "Renamed '%s' to '%s'" % (src, dest)
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             print "No volume named '%s'" % src
             return 1
         print "Unknown Error %s" % e
         return 1
Esempio n. 12
0
 def rename(self, src, dest):
     """ Rename 'src' to 'dest' """
     try:
         helper = self.load_conf(self.config)
         execute('lvrename', helper.volumes.volume_group, src, dest)
         print "Renamed '%s' to '%s'" % (src, dest)
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             print "No volume named '%s'" % src
             return 1
         print "Unknown Error %s" % e
         return 1
Esempio n. 13
0
    def test_execute_nosudo(self):
        execute_args = []

        def mock_popen(args, **kwargs):
            execute_args.extend(args)
            return MockPopen()

        with patch(subprocess, 'Popen', mock_popen):
            utils.execute('ls', '-r', _all=None, color='auto', sudo=False)

        self.assertEqual(4, len(execute_args))
        self.assertEqual(['ls', '-r', '--all', '--color=auto'], execute_args)
Esempio n. 14
0
    def test_execute_nosudo(self):
        execute_args = []

        def mock_popen(args, **kwargs):
            execute_args.extend(args)
            return MockPopen()

        with patch(subprocess, 'Popen', mock_popen):
            utils.execute('ls', '-r', _all=None, color='auto', sudo=False)

        self.assertEqual(4, len(execute_args))
        self.assertEqual(['ls', '-r', '--all', '--color=auto'], execute_args)
Esempio n. 15
0
    def test_export(self):
        # create a volume
        volume = str(uuid4())
        resp = self.put('volumes/%s?size=0' % volume)
        self.assertCode(resp, 200)
        self.assertEquals(resp.body['id'], volume)

        # iscsiadm does not know how to resolve 'localhost'
        # we work around that here
        host_ip = gethostbyname(self.storage.host)
        initiator = find_initiatorname()

        # create the export for some other ip
        resp = self.put('volumes/%s/export' % volume, params={
            'ip': '10.10.10.10',
            'initiator': initiator
        })

        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator discovery should not find our volume
        try:
            out = execute('iscsiadm', mode='discovery', type='sendtargets',
                          portal=host_ip)
            self.assertNotIn(volume, out)
        except ProcessError, e:
            # Newer iscsiadm (than precise) throws an error on no targets
            self.assertEquals(e.errcode, 21)
Esempio n. 16
0
 def lvchange(self, path, **kwargs):
     try:
         return execute('lvchange', path, **kwargs)
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             raise NotFound('No volume named %s.' % id)
         raise
Esempio n. 17
0
 def _do_create(self,
                volume_id,
                size_str,
                tag,
                backup_source_volume_id=None):
     try:
         out = execute('lvcreate',
                       self.volume_group,
                       name=volume_id,
                       size=size_str,
                       addtag=tag)
     except ProcessError, e:
         if not e.errcode == 5 and 'already exists' not in e.err:
             raise
         # We ran out of space on the storage node!
         if "Insufficient free extents" in e.err \
                 or "insufficient free space" in e.err:
             logger.error(e.err)
             raise ServiceUnavailable("LVM reports insufficient "
                                      "free space on drive")
         # If we are requesting a restore, and the existing volume is this
         # same failed restore, it's not an error.
         if backup_source_volume_id and backup_source_volume_id == \
                 self.get(volume_id).get('backup_source_volume_id', False):
             logger.info("Restarting failed restore on '%s'" % volume_id)
         else:
             raise AlreadyExists("Unable to create a new volume named "
                                 "'%s' because one already exists." %
                                 volume_id)
Esempio n. 18
0
 def connect(self):
     """ login or open an iscsi connection """
     try:
         portal = "%s:%s" % (self.ip, self.port)
         execute('iscsiadm',
                 mode='discovery',
                 type='sendtargets',
                 portal=self.ip)
         execute('iscsiadm',
                 mode='node',
                 targetname=self.iqn,
                 portal=self.ip,
                 login=None)
     except ProcessError, e:
         logger.exception("iscsi login failed with '%s'" % e)
         raise ISCSILoginFailed()
Esempio n. 19
0
    def test_export(self):
        # create a volume
        volume = str(uuid4())
        resp = self.put('volumes/%s?size=0' % volume)
        self.assertCode(resp, 200)
        self.assertEquals(resp.body['id'], volume)

        # iscsiadm does not know how to resolve 'localhost'
        # we work around that here
        host_ip = gethostbyname(self.storage.host)
        initiator = find_initiatorname()

        # create the export for some other ip
        resp = self.put('volumes/%s/export' % volume,
                        params={
                            'ip': '10.10.10.10',
                            'initiator': initiator
                        })

        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator discovery should not find our volume
        try:
            out = execute('iscsiadm',
                          mode='discovery',
                          type='sendtargets',
                          portal=host_ip)
            self.assertNotIn(volume, out)
        except ProcessError, e:
            # Newer iscsiadm (than precise) throws an error on no targets
            self.assertEquals(e.errcode, 21)
Esempio n. 20
0
 def lvchange(self, path, **kwargs):
     try:
         return execute('lvchange', path, **kwargs)
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             raise NotFound('No volume named %s.' % id)
         raise
Esempio n. 21
0
 def from_iscsi_connection(id):
     for line in execute('iscsiadm', '-m', 'session').split('\n'):
         if re.search(id, line):
             conn = re.split('\s*', line)
             ip, port, _ = re.split(':|,', conn[2])
             print conn[3], ip, port
             return ISCSIDevice(conn[3], ip, port)
     raise RuntimeError("Unable to find iscsi connection for '%s'" % id)
Esempio n. 22
0
 def run(self, cmd, *args, **kwargs):
     for attempts in range(0, 3):
         try:
             return execute(cmd, *args, **kwargs)
         except ProcessError, e:
             log.error("Command '%s' returned non-zero exit status" % e.cmd)
             log.error("stdout: %s" % e.out)
             log.error("stderr: %s" % e.err)
Esempio n. 23
0
 def from_iscsi_connection(id):
     for line in execute('iscsiadm', '-m', 'session').split('\n'):
         if re.search(id, line):
             conn = re.split('\s*', line)
             ip, port, _ = re.split(':|,', conn[2])
             print conn[3], ip, port
             return ISCSIDevice(conn[3], ip, port)
     raise RuntimeError("Unable to find iscsi connection for '%s'" % id)
Esempio n. 24
0
 def run(self, cmd, *args, **kwargs):
     for attempts in range(0, 3):
         try:
             return execute(cmd, *args, **kwargs)
         except ProcessError, e:
             log.error("Command '%s' returned non-zero exit status" % e.cmd)
             log.error("stdout: %s" % e.out)
             log.error("stderr: %s" % e.err)
Esempio n. 25
0
 def remove(self, path):
     for i in range(0, 10):
         try:
             return execute('lvremove', path, force=None)
         except ProcessError:
             sleep(1)
             continue
     logger.error("Failed to delete volume '%s' after 10 tries" % path)
     raise
Esempio n. 26
0
 def remove(self, path):
     for i in range(0, 10):
         try:
             return execute('lvremove', path, force=None)
         except ProcessError:
             sleep(1)
             continue
     logger.error("Failed to delete volume '%s' after 10 tries" % path)
     raise
Esempio n. 27
0
 def untar_image(self, path, image):
     tarball = os.path.join(path, 'image')
     op_start = time()
     execute('tar', '-C', path, '-zxf', tarball, sudo=False)
     duration = time() - op_start
     mbytes = image.size / 1024 / 1024
     uncompressed = 0
     for f in os.listdir(path):
         # No fair counting twice.
         if f == 'image':
             continue
         fpath = os.path.join(path, f)
         if os.path.isfile(fpath):
             uncompressed += os.path.getsize(fpath)
     uncompressed = uncompressed / 1024 / 1024
     logger.info('STAT: tar %r. Compressed Size: %r MB '
                 'Uncompressed Size: %r MB '
                 'Time: %r Speed: %r' % (image.id, mbytes, uncompressed,
                                         duration, uncompressed / duration))
Esempio n. 28
0
 def _get_volume(self, volume_id):
     volume_name = '%s/%s' % (self.volume_group, volume_id)
     out = execute('lvs',
                   volume_name,
                   noheadings=None,
                   separator=':',
                   units='b',
                   options=','.join(self.LVS_OPTIONS))
     out = out.strip()
     return self._parse_volume(out)
Esempio n. 29
0
 def _scan_volumes(self):
     out = execute('lvs', self.volume_group, noheadings=None, separator=':',
                   units='b', options=','.join(self.LVS_OPTIONS))
     volumes = []
     for line in out.splitlines():
         line = line.strip()
         if not line:
             continue
         volume = self._parse_volume(line)
         volumes.append(volume)
     return volumes
Esempio n. 30
0
 def _scan_volumes(self):
     out = execute('lvs', self.volume_group, noheadings=None, separator=':',
                   units='b', options=','.join(self.LVS_OPTIONS))
     volumes = []
     for line in out.splitlines():
         line = line.strip()
         if not line:
             continue
         volume = self._parse_volume(line)
         volumes.append(volume)
     return volumes
Esempio n. 31
0
 def untar_image(self, path, image):
     tarball = os.path.join(path, 'image')
     op_start = time()
     execute('tar', '-C', path, '-zxf', tarball, sudo=False)
     duration = time() - op_start
     mbytes = image.size / 1024 / 1024
     uncompressed = 0
     for f in os.listdir(path):
         # No fair counting twice.
         if f == 'image':
             continue
         fpath = os.path.join(path, f)
         if os.path.isfile(fpath):
             uncompressed += os.path.getsize(fpath)
     uncompressed = uncompressed / 1024 / 1024
     logger.info('STAT: tar %r. Compressed Size: %r MB '
                 'Uncompressed Size: %r MB '
                 'Time: %r Speed: %r' %
                 (image.id, mbytes, uncompressed,
                  duration, uncompressed / duration))
Esempio n. 32
0
 def status(self):
     options = ('vg_size', 'vg_free', 'lv_count')
     try:
         out = execute('vgs', self.volume_group, noheadings=None, unit='b',
                       options=','.join(options), separator=':')
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             raise ServiceUnavailable("Volume group '%s' not found." %
                                      self.volume_group)
         logger.exception("Unknown error trying to query status of "
                          "volume group '%s'" % self.volume_group)
         raise ServiceUnavailable("[Errno %d] %s" % (e.errcode, e.err))
Esempio n. 33
0
 def status(self):
     options = ('vg_size', 'vg_free', 'lv_count')
     try:
         out = execute('vgs', self.volume_group, noheadings=None, unit='b',
                       options=','.join(options), separator=':')
     except ProcessError, e:
         if e.errcode == 5 and 'not found' in e.err:
             raise ServiceUnavailable("Volume group '%s' not found." %
                                      self.volume_group)
         logger.exception("Unknown error trying to query status of "
                          "volume group '%s'" % self.volume_group)
         raise ServiceUnavailable("[Errno %d] %s" % (e.errcode, e.err))
Esempio n. 34
0
def lvm_capacity():
    """
    Return the total LVM capacity in bytes
    """
    lines = execute('vgdisplay', 'lunr-volume', units='b')
    for line in lines.split('\n'):
        # Find the Volume Group Size in the output
        if re.search('VG Size', line):
            # First convert the bytes to GiB, then round down (trunc)
            # Then convert back to bytes. Do this because lunr/cinder
            # stores the size as an integer not a float
            return bytes_to_gibibytes(int(line.split()[2]))
Esempio n. 35
0
    def test_session(self):
        # Get the session, assert is not 'Connected'
        sessions = self.exports._sessions(self.volume)
        self.assertEquals(len(sessions), 1)
        self.assertEquals(sessions[0]['connected'], False)

        # Connect to the exported volume
        export = self.exports.get(self.volume)
        execute('iscsiadm',
                mode='discovery',
                type='sendtargets',
                portal=self.host)
        execute('iscsiadm',
                mode='node',
                portal=self.host,
                login=None,
                targetname=export['name'])

        # Get the session, assert is 'Connected'
        sessions = self.exports._sessions(self.volume)
        self.assertEquals(len(sessions), 1)
        self.assertEquals(sessions[0]['connected'], True)
        self.assertEquals(sessions[0]['state'], 'active')
        self.assertEquals(sessions[0]['ip'], self.host)
        self.assertTrue('initiator' in sessions[0])

        # Logout of the exported volume
        execute('iscsiadm',
                mode='node',
                portal=self.host,
                logout=None,
                targetname=export['name'])
Esempio n. 36
0
    def get_coalesced_vhd(self, path):
        # Check for old style, image.vhd
        old_style = self.get_oldstyle_vhd(path)
        if old_style:
            return old_style

        op_start = time()

        chain = self.get_vhd_chain(path)
        if len(chain) == 0:
            raise ValueError('Invalid image. Bad vhd chain.')

        journal = os.path.join(path, 'vhdjournal')

        self.reparent_vhd_chain(chain)
        self.repair_vhd_chain(chain)

        while len(chain) > 1:
            child = chain.pop(0)
            parent = chain[0]
            child_size = execute('vhd-util', 'query', '-n', child, '-v',
                                 sudo=False)
            parent_size = execute('vhd-util', 'query', '-n', parent, '-v',
                                  sudo=False)
            if child_size != parent_size:
                execute('vhd-util', 'resize', '-n', parent,
                        '-s', child_size, '-j', journal, sudo=False)
            execute('vhd-util', 'coalesce', '-n', child, sudo=False)

        duration = time() - op_start
        logger.info('STAT: get_coalesced_vhd Time: %r.' % duration)
        return chain[0]
Esempio n. 37
0
    def get_coalesced_vhd(self, path):
        # Check for old style, image.vhd
        old_style = self.get_oldstyle_vhd(path)
        if old_style:
            return old_style

        op_start = time()

        chain = self.get_vhd_chain(path)
        if len(chain) == 0:
            raise ValueError('Invalid image. Bad vhd chain.')

        journal = os.path.join(path, 'vhdjournal')

        self.reparent_vhd_chain(chain)
        self.repair_vhd_chain(chain)

        while len(chain) > 1:
            child = chain.pop(0)
            parent = chain[0]
            child_size = execute('vhd-util', 'query', '-n', child, '-v',
                                 sudo=False)
            parent_size = execute('vhd-util', 'query', '-n', parent, '-v',
                                  sudo=False)
            if child_size != parent_size:
                execute('vhd-util', 'resize', '-n', parent,
                        '-s', child_size, '-j', journal, sudo=False)
            execute('vhd-util', 'coalesce', '-n', child, sudo=False)

        duration = time() - op_start
        logger.info('STAT: get_coalesced_vhd Time: %r.' % duration)
        return chain[0]
Esempio n. 38
0
 def test_export(self):
     volume_id = str(uuid4())
     # Create volume.
     path = self.project_id + '/volumes/%s' % volume_id
     resp = self.request(path, 'PUT', {
         'size': 1, 'volume_type_name': self.volume_type})
     self.assert200(resp.code, resp.body, 'create volume')
     self.assertEquals(resp.body['id'], volume_id)
     fake_initiator = 'monkeys'
     # Create export
     path = path + '/export'
     resp = self.request(path, 'PUT', {'ip': '127.0.0.1',
                                       'initiator': fake_initiator})
     target = resp.body['target_name']
     host = gethostbyname(self.api.host)
     self.assert200(resp.code, resp.body, 'create export')
     self.assertEquals(resp.body['status'], 'ATTACHING')
     self.assertEquals(resp.body['id'], volume_id)
     # Attach!
     execute('iscsiadm', mode='discovery', type='sendtargets',
             portal=host)
     execute('iscsiadm', mode='node', targetname=target,
             portal=host, login=None)
     resp = self.request(path, 'POST', {'status': 'ATTACHED',
                                        'mountpoint': '/some/thing',
                                        'instance_id': 'foo'})
     # Detach!
     execute('iscsiadm', mode='node', targetname=target,
             portal=host, logout=None)
     self.assert200(resp.code, resp.body, 'update export')
     self.assertNotEquals(resp.body['session_ip'], '')
     self.assertNotEquals(resp.body['session_initiator'], '')
     self.assertEquals(resp.body['status'], 'ATTACHED')
     self.assertEquals(resp.body['mountpoint'], '/some/thing')
     self.assertEquals(resp.body['instance_id'], 'foo')
Esempio n. 39
0
 def test_export(self):
     volume_id = str(uuid4())
     # Create volume.
     path = self.project_id + "/volumes/%s" % volume_id
     resp = self.request(path, "PUT", {"size": 1, "volume_type_name": self.volume_type})
     self.assert200(resp.code, resp.body, "create volume")
     self.assertEquals(resp.body["id"], volume_id)
     fake_initiator = "monkeys"
     # Create export
     path = path + "/export"
     resp = self.request(path, "PUT", {"ip": "127.0.0.1", "initiator": fake_initiator})
     target = resp.body["target_name"]
     host = gethostbyname(self.api.host)
     self.assert200(resp.code, resp.body, "create export")
     self.assertEquals(resp.body["status"], "ATTACHING")
     self.assertEquals(resp.body["id"], volume_id)
     # Attach!
     execute("iscsiadm", mode="discovery", type="sendtargets", portal=host)
     execute("iscsiadm", mode="node", targetname=target, portal=host, login=None)
     resp = self.request(path, "POST", {"status": "ATTACHED", "mountpoint": "/some/thing", "instance_id": "foo"})
     # Detach!
     execute("iscsiadm", mode="node", targetname=target, portal=host, logout=None)
     self.assert200(resp.code, resp.body, "update export")
     self.assertNotEquals(resp.body["session_ip"], "")
     self.assertNotEquals(resp.body["session_initiator"], "")
     self.assertEquals(resp.body["status"], "ATTACHED")
     self.assertEquals(resp.body["mountpoint"], "/some/thing")
     self.assertEquals(resp.body["instance_id"], "foo")
Esempio n. 40
0
    def clone(self, id=None, src=None, backup=None, size=None):
        """
        This runs a clone job outside of the storage api,
        which is useful for performance testing backup restores
        (Example: storage tools clone volume-clone
          --backup volume-backup --src volume-original)
        """
        # Set basic Logging
        logging.basicConfig()
        # Get the lunr logger
        log = logger.get_logger()
        # Output Debug level info
        log.logger.setLevel(logging.DEBUG)
        # Load the local storage configuration
        conf = LunrConfig.from_storage_conf()
        # Init the volume helper
        volume = VolumeHelper(conf)

        # Attempt to figure out the original volume size
        size = size or str(volume.get(src)['size'] / 1073741824)
        # Size is in gigs
        if not re.match('G', size):
            size = size + 'G'
        # Create a tag to apply to the lvm volume
        tag = encode_tag(source_volume_id=src, backup_id=backup)
        # Create the volume
        execute('lvcreate',
                volume.volume_group,
                name=id,
                size=size,
                addtag=tag)
        # Get info for the newly created volume
        new = volume.get(id)

        with self.timeit():
            print("Starting Backup")
            # Restore volume from the backup
            volume.clone(new, src, backup)
Esempio n. 41
0
    def test_delete_while_attached(self):
        volume_id = str(uuid4())
        host = gethostbyname(self.api.host)
        # Create a new volume
        path = 'test/volumes/%s' % volume_id
        resp = self.put(
            path, params={'size': 1, 'volume_type_name': self.volume_type})
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body['id'], volume_id)

        path = path + '/export'
        resp = self.put(path, params={'ip': '127.0.0.1',
                                      'initiator': 'fake_initiator'})
        self.assertEquals(resp.code, 200)
        target = resp.body['target_name']

        # Connect to the volume
        execute('iscsiadm', mode='discovery', type='sendtargets',
                portal=host)
        execute('iscsiadm', mode='node', targetname=target,
                portal=host, login=None)

        # Attempt to delete the in-use volume
        resp = self.delete('test/volumes/%s' % volume_id)
        self.assertEquals(resp.code, 409)
        self.assertEquals(resp.reason, 'Conflict')
        self.assertIn("Cannot delete '%s' while export in use" % volume_id,
                      resp.body['reason'])

        # Assert the volume is still active
        resp = self.get('test/volumes/%s' % volume_id)
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body['status'], 'ACTIVE')

        # Logout of the exported volume
        execute('iscsiadm', mode='node', targetname=target,
                portal=host, logout=None)

        # Attempt to delete the volume while not in-use
        resp = self.delete('test/volumes/%s' % volume_id)
        self.assertEquals(resp.code, 200)

        # Volume should be deleting
        resp = self.get('test/volumes/%s' % volume_id)
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body['status'], 'DELETING')

        # API Should eventually report the volume as 'DELETED'
        for i in range(0, 30):
            time.sleep(2)
            resp = self.get('test/volumes/%s' % volume_id)
            self.assertEquals(resp.code, 200)
            if resp.body['status'] == 'DELETED':
                self.assert_(True)
                return
        self.fail("test/volumes/%s never returned a 'status' of 'DELETED'" %
                  volume_id)
Esempio n. 42
0
    def test_export_force_delete(self):
        # create a volume
        volume = str(uuid4())
        resp = self.put('volumes/%s?size=0' % volume)
        self.assertCode(resp, 200)
        self.assertEquals(resp.body['id'], volume)

        # iscsiadm does not know how to resolve 'localhost'
        # we work around that here
        host_ip = gethostbyname(self.storage.host)

        # create the export
        resp = self.put('volumes/%s/export' % volume,
                        params={
                            'ip': host_ip,
                            'initiator': 'fake_initiator'
                        })
        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator login
        execute('iscsiadm',
                mode='discovery',
                type='sendtargets',
                portal=host_ip)
        execute('iscsiadm',
                mode='node',
                targetname=target,
                portal=host_ip,
                login=None)

        resp = self.get('volumes/%s/export' % volume)
        self.assertEquals(resp.body['sessions'][0]['connected'], True)

        # force delete the export
        resp = self.delete('volumes/%s/export' % volume,
                           params={'force': True})
        self.assertCode(resp, 200)

        resp = self.get('volumes/%s/export' % volume)
        self.assertEquals(resp.body.get('sessions', []), [])

        # iscsi initator logout
        execute('iscsiadm',
                mode='node',
                targetname=target,
                portal=host_ip,
                logout=None)

        # delete the volume
        resp = self.delete('volumes/%s' % volume)
        self.assertCode(resp, 200)
Esempio n. 43
0
    def copy_image(self, volume, image, glance, tmp_vol, scrub_callback):
        logger.rename('lunr.storage.helper.volume.copy_image')
        setproctitle("lunr-copy-image: " + volume['id'])
        copy_image_start = time()
        convert_dir = None
        try:
            if image.disk_format == 'raw':
                self.write_raw_image(glance, image, volume['path'])
                return

            convert_dir = self.prepare_tmp_vol(tmp_vol)

            if not os.path.exists(convert_dir):
                raise ValueError("Convert dir doesn't exist!")

            try:
                path = mkdtemp(dir=convert_dir)
                logger.info("Image convert tmp dir: %s" % path)
                image_file = os.path.join(path, 'image')
                self.write_raw_image(glance, image, image_file)

                if (image.disk_format == 'vhd' and
                        image.container_format == 'ovf'):
                    self.untar_image(path, image)
                    image_file = self.get_coalesced_vhd(path)

                op_start = time()
                out = execute('qemu-img', 'convert', '-O', 'raw', image_file,
                              volume['path'])
                duration = time() - op_start
                mbytes = os.path.getsize(image_file) / 1024 / 1024
                logger.info('STAT: image convert %r. Image Size: %r MB '
                            'Time: %r Speed: %r' %
                            (image.id, mbytes, duration, mbytes / duration))
            except Exception, e:
                logger.exception("Exception in image conversion")
                raise

        except Exception, e:
            # We have to clean this up no matter what happened.
            # Delete volume syncronously. Clean up db in callback.
            logger.exception('Unhandled exception in copy_image')
            self.remove_lvm_volume(volume)
Esempio n. 44
0
    def copy_image(self, volume, image, glance, tmp_vol, scrub_callback):
        logger.rename('lunr.storage.helper.volume.copy_image')
        setproctitle("lunr-copy-image: " + volume['id'])
        copy_image_start = time()
        convert_dir = None
        try:
            if image.disk_format == 'raw':
                self.write_raw_image(glance, image, volume['path'])
                return

            convert_dir = self.prepare_tmp_vol(tmp_vol)

            if not os.path.exists(convert_dir):
                raise ValueError("Convert dir doesn't exist!")

            try:
                path = mkdtemp(dir=convert_dir)
                logger.info("Image convert tmp dir: %s" % path)
                image_file = os.path.join(path, 'image')
                self.write_raw_image(glance, image, image_file)

                if (image.disk_format == 'vhd'
                        and image.container_format == 'ovf'):
                    self.untar_image(path, image)
                    image_file = self.get_coalesced_vhd(path)

                op_start = time()
                out = execute('qemu-img', 'convert', '-O', 'raw', image_file,
                              volume['path'])
                duration = time() - op_start
                mbytes = os.path.getsize(image_file) / 1024 / 1024
                logger.info('STAT: image convert %r. Image Size: %r MB '
                            'Time: %r Speed: %r' %
                            (image.id, mbytes, duration, mbytes / duration))
            except Exception, e:
                logger.exception("Exception in image conversion")
                raise

        except Exception, e:
            # We have to clean this up no matter what happened.
            # Delete volume syncronously. Clean up db in callback.
            logger.exception('Unhandled exception in copy_image')
            self.remove_lvm_volume(volume)
Esempio n. 45
0
    def test_delete_while_attached(self):
        volume_id = str(uuid4())
        host = gethostbyname(self.api.host)
        # Create a new volume
        path = "test/volumes/%s" % volume_id
        resp = self.put(path, params={"size": 1, "volume_type_name": self.volume_type})
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body["id"], volume_id)

        path = path + "/export"
        resp = self.put(path, params={"ip": "127.0.0.1", "initiator": "fake_initiator"})
        self.assertEquals(resp.code, 200)
        target = resp.body["target_name"]

        # Connect to the volume
        execute("iscsiadm", mode="discovery", type="sendtargets", portal=host)
        execute("iscsiadm", mode="node", targetname=target, portal=host, login=None)

        # Attempt to delete the in-use volume
        resp = self.delete("test/volumes/%s" % volume_id)
        self.assertEquals(resp.code, 409)
        self.assertEquals(resp.reason, "Conflict")
        self.assertIn("Cannot delete '%s' while export in use" % volume_id, resp.body["reason"])

        # Assert the volume is still active
        resp = self.get("test/volumes/%s" % volume_id)
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body["status"], "ACTIVE")

        # Logout of the exported volume
        execute("iscsiadm", mode="node", targetname=target, portal=host, logout=None)

        # Attempt to delete the volume while not in-use
        resp = self.delete("test/volumes/%s" % volume_id)
        self.assertEquals(resp.code, 200)

        # Volume should be deleting
        resp = self.get("test/volumes/%s" % volume_id)
        self.assertEquals(resp.code, 200)
        self.assertEquals(resp.body["status"], "DELETING")

        # API Should eventually report the volume as 'DELETED'
        for i in range(0, 30):
            time.sleep(2)
            resp = self.get("test/volumes/%s" % volume_id)
            self.assertEquals(resp.code, 200)
            if resp.body["status"] == "DELETED":
                self.assert_(True)
                return
        self.fail("test/volumes/%s never returned a 'status' of 'DELETED'" % volume_id)
Esempio n. 46
0
    def prepare_tmp_vol(self, tmp_vol):
        if not tmp_vol:
            raise ValueError("No tmp_vol")
        if not os.path.exists(tmp_vol['path']):
            raise ValueError("tmp_vol doesn't exist")

        if self.has_old_mkfs:
            execute('/sbin/mkfs.ext4', tmp_vol['path'], sudo=False)
        else:
            execute('/sbin/mkfs.ext4', '-E', 'root_owner',  tmp_vol['path'],
                    sudo=False)

        mount_dir = mkdtemp()
        execute('mount', '-t', 'ext4', '-o', 'loop', tmp_vol['path'],
                mount_dir)
        return mount_dir
Esempio n. 47
0
    def prepare_tmp_vol(self, tmp_vol):
        if not tmp_vol:
            raise ValueError("No tmp_vol")
        if not os.path.exists(tmp_vol['path']):
            raise ValueError("tmp_vol doesn't exist")

        if self.has_old_mkfs:
            execute('/sbin/mkfs.ext4', tmp_vol['path'], sudo=False)
        else:
            execute('/sbin/mkfs.ext4', '-E', 'root_owner',  tmp_vol['path'],
                    sudo=False)

        mount_dir = mkdtemp()
        execute('mount', '-t', 'ext4', '-o', 'loop', tmp_vol['path'],
                mount_dir)
        return mount_dir
Esempio n. 48
0
 def test_export(self):
     volume_id = str(uuid4())
     # Create volume.
     path = self.project_id + '/volumes/%s' % volume_id
     resp = self.request(path, 'PUT', {
         'size': 1,
         'volume_type_name': self.volume_type
     })
     self.assert200(resp.code, resp.body, 'create volume')
     self.assertEquals(resp.body['id'], volume_id)
     fake_initiator = 'monkeys'
     # Create export
     path = path + '/export'
     resp = self.request(path, 'PUT', {
         'ip': '127.0.0.1',
         'initiator': fake_initiator
     })
     target = resp.body['target_name']
     host = gethostbyname(self.api.host)
     self.assert200(resp.code, resp.body, 'create export')
     self.assertEquals(resp.body['status'], 'ATTACHING')
     self.assertEquals(resp.body['id'], volume_id)
     # Attach!
     execute('iscsiadm', mode='discovery', type='sendtargets', portal=host)
     execute('iscsiadm',
             mode='node',
             targetname=target,
             portal=host,
             login=None)
     resp = self.request(
         path, 'POST', {
             'status': 'ATTACHED',
             'mountpoint': '/some/thing',
             'instance_id': 'foo'
         })
     # Detach!
     execute('iscsiadm',
             mode='node',
             targetname=target,
             portal=host,
             logout=None)
     self.assert200(resp.code, resp.body, 'update export')
     self.assertNotEquals(resp.body['session_ip'], '')
     self.assertNotEquals(resp.body['session_initiator'], '')
     self.assertEquals(resp.body['status'], 'ATTACHED')
     self.assertEquals(resp.body['mountpoint'], '/some/thing')
     self.assertEquals(resp.body['instance_id'], 'foo')
Esempio n. 49
0
 def _do_create(self, volume_id, size_str, tag,
                backup_source_volume_id=None):
     try:
         out = execute('lvcreate', self.volume_group,
                       name=volume_id, size=size_str, addtag=tag)
     except ProcessError, e:
         if not e.errcode == 5 and 'already exists' not in e.err:
             raise
         # We ran out of space on the storage node!
         if "Insufficient free extents" in e.err \
                 or "insufficient free space" in e.err:
             logger.error(e.err)
             raise ServiceUnavailable("LVM reports insufficient "
                                      "free space on drive")
         # If we are requesting a restore, and the existing volume is this
         # same failed restore, it's not an error.
         if backup_source_volume_id and backup_source_volume_id == \
                 self.get(volume_id).get('backup_source_volume_id', False):
             logger.info("Restarting failed restore on '%s'" % volume_id)
         else:
             raise AlreadyExists("Unable to create a new volume named "
                                 "'%s' because one already exists." %
                                 volume_id)
Esempio n. 50
0
    def test_export_force_delete(self):
        # create a volume
        volume = str(uuid4())
        resp = self.put('volumes/%s?size=0' % volume)
        self.assertCode(resp, 200)
        self.assertEquals(resp.body['id'], volume)

        # iscsiadm does not know how to resolve 'localhost'
        # we work around that here
        host_ip = gethostbyname(self.storage.host)

        # create the export
        resp = self.put('volumes/%s/export' % volume, params={
            'ip': host_ip,
            'initiator': 'fake_initiator'
        })
        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator login
        execute('iscsiadm', mode='discovery', type='sendtargets',
                portal=host_ip)
        execute('iscsiadm', mode='node', targetname=target,
                portal=host_ip, login=None)

        resp = self.get('volumes/%s/export' % volume)
        self.assertEquals(resp.body['sessions'][0]['connected'], True)

        # force delete the export
        resp = self.delete('volumes/%s/export' % volume,
                           params={'force': True})
        self.assertCode(resp, 200)

        resp = self.get('volumes/%s/export' % volume)
        self.assertEquals(resp.body.get('sessions', []), [])

        # iscsi initator logout
        execute('iscsiadm', mode='node', targetname=target,
                portal=host_ip, logout=None)

        # delete the volume
        resp = self.delete('volumes/%s' % volume)
        self.assertCode(resp, 200)
Esempio n. 51
0
    def test_session(self):
        # Get the session, assert is not 'Connected'
        sessions = self.exports._sessions(self.volume)
        self.assertEquals(len(sessions), 1)
        self.assertEquals(sessions[0]['connected'], False)

        # Connect to the exported volume
        export = self.exports.get(self.volume)
        execute('iscsiadm', mode='discovery', type='sendtargets',
                portal=self.host)
        execute('iscsiadm', mode='node', portal=self.host, login=None,
                targetname=export['name'])

        # Get the session, assert is 'Connected'
        sessions = self.exports._sessions(self.volume)
        self.assertEquals(len(sessions), 1)
        self.assertEquals(sessions[0]['connected'], True)
        self.assertEquals(sessions[0]['state'], 'active')
        self.assertEquals(sessions[0]['ip'], self.host)
        self.assertTrue('initiator' in sessions[0])

        # Logout of the exported volume
        execute('iscsiadm', mode='node', portal=self.host, logout=None,
                targetname=export['name'])
Esempio n. 52
0
 def dm_device(pid):
     for line in execute('lsof', '-p', str(pid)).split('\n'):
         if re.search('/dev/dm-', line):
             return re.split('\s*', line)[8].lstrip('/dev/')
     raise RuntimeError("DM for lunr-clone pid '%s' not found" % pid)
Esempio n. 53
0
 def test_writable_cow_multiline_table(self):
     # Let's do some silly math
     size = directio.size(self._ramdisk)
     megs = size / 1024 / 1024
     megs = megs - megs % 4
     # 12 megs for a volume, 4 for lvm itself
     alloc = megs - 12 - 4
     vg = self.conf.string('volume', 'volume_group', None)
     # Reserve a 4m hole at the front, and 8m at the end
     execute('lvcreate', vg, size='4m', name='tmpvol')
     execute('lvcreate', vg, size='%sm' % alloc, name='wasted')
     execute('lvremove', '%s/tmpvol' % vg, force=None)
     foo = execute('pvs', self._ramdisk)
     foo = execute('vgs', vg)
     foo = execute('lvs', vg)
     volume_id = str(uuid4())
     self.volume.create(volume_id)
     volume = self.volume.get(volume_id)
     execute('lvremove', '%s/wasted' % vg, force=None)
     dmname = '%s-%s' % (re.sub('-', '--', vg),
                         re.sub('-', '--', volume_id))
     foo = execute('dmsetup', 'table', dmname)
     self.assert_('\n' in foo)
     backup_id = str(uuid4())
     snapshot = self.volume.create_snapshot(volume_id, backup_id, '123456')
     scrub = Scrub(LunrConfig())
     (cow_name, cow_path) = scrub.get_writable_cow(snapshot, volume)
     execute('dmsetup', 'remove', cow_name)
     self.assertTrue(True)
Esempio n. 54
0
 def clone_pid(id):
     for line in execute('ps', 'aux').split('\n'):
         if re.search('lunr-clone.*%s' % id, line):
             return int(re.split('\s*', line)[1])
     raise RuntimeError("Unable to find pid for lunr-clone '%s'" % id)
Esempio n. 55
0
def ietadm(*args, **kwargs):
    try:
        return execute('ietadm', *args, **kwargs)
    except ProcessError, e:
        raise IscsitargetError.get_exc(e)
Esempio n. 56
0
    def test_export(self):
        # create a volume
        volume = str(uuid4())
        resp = self.put('volumes/%s?size=0' % volume)
        self.assertCode(resp, 200)
        self.assertEquals(resp.body['id'], volume)

        # iscsiadm does not know how to resolve 'localhost'
        # we work around that here
        host_ip = gethostbyname(self.storage.host)
        initiator = find_initiatorname()

        # create the export for some other ip
        resp = self.put('volumes/%s/export' % volume, params={
            'ip': '10.10.10.10',
            'initiator': initiator
        })

        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator discovery should not find our volume
        out = execute('iscsiadm', mode='discovery', type='sendtargets',
                      portal=host_ip)

        self.assertNotIn(volume, out)

        # clean up
        self.delete('volumes/%s/export' % volume)

        # create the export for our ip
        resp = self.put('volumes/%s/export' % volume, params={
            'ip': host_ip,
            'initiator': initiator
        })
        self.assertCode(resp, 200)
        target = resp.body['name']
        self.assert_(volume in target)

        # iscsi initator login
        out = execute('iscsiadm', mode='discovery', type='sendtargets',
                      portal=host_ip)

        # Now discovery certainly *should* show our volume.
        self.assertIn(volume, out)

        execute('iscsiadm', mode='node', targetname=target,
                portal=host_ip, login=None)

        # Export should show attached
        resp = self.get('volumes/%s/export' % volume)
        self.assertEquals(resp.body['sessions'][0]['connected'], True)

        # find device
        device = find_device(host_ip, target)

        # Block device should work as expected
        data = "IT JUST GOT REAL!"
        with open(device, 'wb') as file:
            file.write(data)

        with open(device, 'rb') as file:
            input = file.read(len(data))
            self.assertEquals(data, input)

        # Shouldn't be able to delete export while attached with no intitator
        resp = self.delete('volumes/%s/export' % volume)
        self.assertEquals(resp.code, 409)

        # Shouldn't be able to delete export while attached with our initiator
        initiator = find_initiatorname()
        resp = self.delete('volumes/%s/export?initiator=%s' %
                           (volume, initiator))
        self.assertEquals(resp.code, 409)

        # Should be able to "delete" export for a nonattached initiator
        resp = self.delete('volumes/%s/export?initiator=%s' %
                           (volume, 'someotheriniator'))
        self.assertEquals(resp.code // 100, 2)

        # iscsi initator logout
        execute('iscsiadm', mode='node', targetname=target, portal=host_ip,
                logout=None)

        # *Should* be able to delete export after detaching
        resp = self.delete('volumes/%s/export?initiator=%s' %
                           (volume, 'someotheriniator'))
        self.assertEquals(resp.code // 100, 2)

        # Should now 404
        resp = self.delete('volumes/%s/export' % volume)
        self.assertEquals(resp.code, 404)

        # delete the volume
        resp = self.delete('volumes/%s' % volume)
        self.assertEquals(resp.code // 100, 2)
Esempio n. 57
0
def temporary_mount(device):
    with TemporaryDirectory() as mount_path:
        execute('mount', device, mount_path)
        yield mount_path
        # leave a message in the bottle
        execute('umount', mount_path)