class StorageServersModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.pool = StoragePoolModel(**kargs)
def get_list(self, _target_type=None):
if not _target_type:
target_type = STORAGE_SOURCES.keys()
else:
target_type = [_target_type]
pools = self.pools.get_list()
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
server_list = []
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if (pool_info['type'] in target_type and
pool_info['source']['addr'] not in server_list):
# Avoid to add same server for multiple times
# if it hosts more than one storage type
server_list.append(pool_info['source']['addr'])
except NotFoundError:
pass
return server_list
class StorageServerModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.pool = StoragePoolModel(**kargs)
def lookup(self, server):
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if (pool_info['type'] in STORAGE_SERVERS and
pool_info['source']['addr'] == server):
info = dict(host=server)
if (pool_info['type'] == "iscsi" and
'port' in pool_info['source']):
info["port"] = pool_info['source']['port']
return info
except NotFoundError:
# Avoid inconsistent pool result because of lease between list
# lookup
pass
raise NotFoundError("KCHSR0001E", {'server': server})
class StorageServersModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.pool = StoragePoolModel(**kargs)
def get_list(self, _target_type=None):
if not _target_type:
target_type = STORAGE_SOURCES.keys()
else:
target_type = [_target_type]
pools = self.pools.get_list()
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
server_list = []
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if (pool_info['type'] in target_type and
pool_info['source']['addr'] not in server_list):
# Avoid to add same server for multiple times
# if it hosts more than one storage type
server_list.append(pool_info['source']['addr'])
except NotFoundError:
pass
return server_list
def _create_volume_with_url(self, cb, params):
pool_name = params['pool']
name = params['name']
url = params['url']
pool_model = StoragePoolModel(conn=self.conn,
objstore=self.objstore)
pool = pool_model.lookup(pool_name)
file_path = os.path.join(pool['path'], name)
with contextlib.closing(urllib2.urlopen(url)) as response:
with open(file_path, 'w') as volume_file:
remote_size = response.info().getheader('Content-Length', '-')
downloaded_size = 0
try:
while True:
chunk_data = response.read(READ_CHUNK_SIZE)
if not chunk_data:
break
volume_file.write(chunk_data)
downloaded_size += len(chunk_data)
cb('%s/%s' % (downloaded_size, remote_size))
except Exception, e:
if os.path.isfile(file_path):
os.remove(file_path)
raise OperationFailed('KCHVOL0007E', {'name': name,
'pool': pool_name,
'err': e.message})
class StorageServerModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.pool = StoragePoolModel(**kargs)
def lookup(self, server):
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if (pool_info['type'] in STORAGE_SERVERS
and pool_info['source']['addr'] == server):
info = dict(host=server)
if (pool_info['type'] == "iscsi"
and 'port' in pool_info['source']):
info["port"] = pool_info['source']['port']
return info
except NotFoundError:
# Avoid inconsistent pool result because of lease between list
# lookup
pass
raise NotFoundError("KCHSR0001E", {'server': server})
class StorageServerModel(object):
def __init__(self, **kargs):
self.conn = kargs["conn"]
self.pool = StoragePoolModel(**kargs)
def lookup(self, server):
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if pool_info["source"] and pool_info["source"]["addr"] == server:
return dict(host=server)
except NotFoundError:
# Avoid inconsistent pool result because of lease between list
# lookup
pass
raise NotFoundError("KCHSR0001E", {"server": server})
class StorageServerModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.pool = StoragePoolModel(**kargs)
def lookup(self, server):
conn = self.conn.get()
pools = conn.listStoragePools()
pools += conn.listDefinedStoragePools()
for pool in pools:
try:
pool_info = self.pool.lookup(pool)
if pool_info['source'] and \
pool_info['source']['addr'] == server:
return dict(host=server)
except NotFoundError:
# Avoid inconsistent pool result because of lease between list
# lookup
pass
raise NotFoundError('server %s does not used by kimchi' % server)
class StorageVolumeModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.objstore = kargs['objstore']
self.task = TaskModel(**kargs)
self.storagevolumes = StorageVolumesModel(**kargs)
self.storagepool = StoragePoolModel(**kargs)
@staticmethod
def get_storagevolume(poolname, name, conn):
pool = StoragePoolModel.get_storagepool(poolname, conn)
if not pool.isActive():
raise InvalidOperation("KCHVOL0006E", {'name': pool})
try:
return pool.storageVolLookupByName(name.encode("utf-8"))
except libvirt.libvirtError as e:
if e.get_error_code() == libvirt.VIR_ERR_NO_STORAGE_VOL:
raise NotFoundError("KCHVOL0002E", {'name': name,
'pool': poolname})
else:
raise
def lookup(self, pool, name):
vol = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
path = vol.path()
info = vol.info()
xml = vol.XMLDesc(0)
try:
fmt = xpath_get_text(xml, "/volume/target/format/@type")[0]
except IndexError:
# Not all types of libvirt storage can provide volume format
# infomation. When there is no format information, we assume
# it's 'raw'.
fmt = 'raw'
iso_img = None
# 'raw' volumes from 'logical' pools may actually be 'iso';
# libvirt always reports them as 'raw'
pool_info = self.storagepool.lookup(pool)
if pool_info['type'] == 'logical' and fmt == 'raw':
try:
iso_img = IsoImage(path)
except IsoFormatError:
# not 'iso' afterall
pass
else:
fmt = 'iso'
used_by = get_disk_used_by(self.objstore, self.conn, path)
res = dict(type=VOLUME_TYPE_MAP[info[0]],
capacity=info[1],
allocation=info[2],
path=path,
used_by=used_by,
format=fmt)
if fmt == 'iso':
if os.path.islink(path):
path = os.path.join(os.path.dirname(path), os.readlink(path))
os_distro = os_version = 'unknown'
try:
if iso_img is None:
iso_img = IsoImage(path)
os_distro, os_version = iso_img.probe()
bootable = True
except IsoFormatError:
bootable = False
res.update(
dict(os_distro=os_distro, os_version=os_version, path=path,
bootable=bootable))
return res
def wipe(self, pool, name):
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
try:
volume.wipePattern(libvirt.VIR_STORAGE_VOL_WIPE_ALG_ZERO, 0)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0009E",
{'name': name, 'err': e.get_error_message()})
def delete(self, pool, name):
pool_info = StoragePoolModel(conn=self.conn,
objstore=self.objstore).lookup(pool)
if pool_info['type'] in READONLY_POOL_TYPE:
raise InvalidParameter("KCHVOL0012E", {'type': pool_info['type']})
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
try:
volume.delete(0)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0010E",
{'name': name, 'err': e.get_error_message()})
def resize(self, pool, name, size):
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
# When decreasing the storage volume capacity, the flag
# VIR_STORAGE_VOL_RESIZE_SHRINK must be used
flags = 0
if volume.info()[1] > size:
# FIXME: Even using VIR_STORAGE_VOL_RESIZE_SHRINK flag it is not
# possible to decrease the volume capacity due a libvirt bug
# For reference:
# - https://bugzilla.redhat.com/show_bug.cgi?id=1021802
flags = libvirt.VIR_STORAGE_VOL_RESIZE_SHRINK
try:
volume.resize(size, flags)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0011E",
{'name': name, 'err': e.get_error_message()})
def clone(self, pool, name, new_pool=None, new_name=None):
"""Clone a storage volume.
Arguments:
pool -- The name of the original pool.
name -- The name of the original volume.
new_pool -- The name of the destination pool (optional). If omitted,
the new volume will be created on the same pool as the
original one.
new_name -- The name of the new volume (optional). If omitted, a new
value based on the original volume's name will be used.
Return:
A Task running the clone operation.
"""
# the same pool will be used if no pool is specified
if new_pool is None:
new_pool = pool
# a default name based on the original name will be used if no name
# is specified
if new_name is None:
base, ext = os.path.splitext(name)
new_name = get_next_clone_name(self.storagevolumes.get_list(pool),
base, ext)
params = {'pool': pool,
'name': name,
'new_pool': new_pool,
'new_name': new_name}
taskid = add_task(u'/storagepools/%s/storagevolumes/%s' %
(pool, new_name), self._clone_task, self.objstore,
params)
return self.task.lookup(taskid)
def _clone_task(self, cb, params):
"""Asynchronous function which performs the clone operation.
This function copies all the data inside the original volume into the
new one.
Arguments:
cb -- A callback function to signal the Task's progress.
params -- A dict with the following values:
"pool": The name of the original pool.
"name": The name of the original volume.
"new_pool": The name of the destination pool.
"new_name": The name of the new volume.
"""
orig_pool_name = params['pool']
orig_vol_name = params['name']
new_pool_name = params['new_pool']
new_vol_name = params['new_name']
try:
cb('setting up volume cloning')
orig_vir_vol = StorageVolumeModel.get_storagevolume(orig_pool_name,
orig_vol_name,
self.conn)
orig_vol = self.lookup(orig_pool_name, orig_vol_name)
new_vir_pool = StoragePoolModel.get_storagepool(new_pool_name,
self.conn)
cb('building volume XML')
root_elem = E.volume()
root_elem.append(E.name(new_vol_name))
root_elem.append(E.capacity(unicode(orig_vol['capacity']),
unit='bytes'))
target_elem = E.target()
target_elem.append(E.format(type=orig_vol['format']))
root_elem.append(target_elem)
new_vol_xml = ET.tostring(root_elem, encoding='utf-8',
pretty_print=True)
cb('cloning volume')
new_vir_pool.createXMLFrom(new_vol_xml, orig_vir_vol, 0)
except (InvalidOperation, NotFoundError, libvirt.libvirtError), e:
raise OperationFailed('KCHVOL0023E',
{'name': orig_vol_name,
'pool': orig_pool_name,
'err': e.get_error_message()})
new_vol = self.lookup(new_pool_name, new_vol_name)
cb('adding volume to the object store')
set_disk_used_by(self.objstore, new_vol['path'], [])
cb('OK', True)
def _create_volume_with_url(self, cb, params):
pool_name = params['pool']
name = params['name']
url = params['url']
pool_model = StoragePoolModel(conn=self.conn,
objstore=self.objstore)
pool = pool_model.lookup(pool_name)
if pool['type'] in ['dir', 'netfs']:
file_path = os.path.join(pool['path'], name)
else:
file_path = tempfile.mkstemp(prefix=name)[1]
with contextlib.closing(urllib2.urlopen(url)) as response:
with open(file_path, 'w') as volume_file:
remote_size = response.info().getheader('Content-Length', '-')
downloaded_size = 0
try:
while True:
chunk_data = response.read(READ_CHUNK_SIZE)
if not chunk_data:
break
volume_file.write(chunk_data)
downloaded_size += len(chunk_data)
cb('%s/%s' % (downloaded_size, remote_size))
except (IOError, libvirt.libvirtError) as e:
if os.path.isfile(file_path):
os.remove(file_path)
raise OperationFailed('KCHVOL0007E', {'name': name,
'pool': pool_name,
'err': e.message})
if pool['type'] in ['dir', 'netfs']:
virt_pool = StoragePoolModel.get_storagepool(pool_name, self.conn)
virt_pool.refresh(0)
else:
def _stream_handler(stream, nbytes, fd):
return fd.read(nbytes)
virt_stream = virt_vol = None
try:
task = self.create(pool_name, {'name': name,
'format': 'raw',
'capacity': downloaded_size,
'allocation': downloaded_size})
self.task.wait(task['id'])
virt_vol = StorageVolumeModel.get_storagevolume(pool_name,
name,
self.conn)
virt_stream = self.conn.get().newStream(0)
virt_vol.upload(virt_stream, 0, downloaded_size, 0)
with open(file_path) as fd:
virt_stream.sendAll(_stream_handler, fd)
virt_stream.finish()
except (IOError, libvirt.libvirtError) as e:
try:
if virt_stream:
virt_stream.abort()
if virt_vol:
virt_vol.delete(0)
except libvirt.libvirtError, virt_e:
kimchi_log.error(virt_e.message)
finally:
raise OperationFailed('KCHVOL0007E', {'name': name,
'pool': pool_name,
'err': e.message})
finally:
def _create_volume_with_url(self, cb, params):
pool_name = params['pool']
name = params['name']
url = params['url']
pool_model = StoragePoolModel(conn=self.conn, objstore=self.objstore)
pool = pool_model.lookup(pool_name)
if pool['type'] in ['dir', 'netfs']:
file_path = os.path.join(pool['path'], name)
else:
file_path = tempfile.mkstemp(prefix=name)[1]
with contextlib.closing(urllib2.urlopen(url)) as response:
with open(file_path, 'w') as volume_file:
remote_size = response.info().getheader('Content-Length', '-')
downloaded_size = 0
try:
while True:
chunk_data = response.read(READ_CHUNK_SIZE)
if not chunk_data:
break
volume_file.write(chunk_data)
downloaded_size += len(chunk_data)
cb('%s/%s' % (downloaded_size, remote_size))
except (IOError, libvirt.libvirtError) as e:
if os.path.isfile(file_path):
os.remove(file_path)
raise OperationFailed('KCHVOL0007E', {
'name': name,
'pool': pool_name,
'err': e.message
})
if pool['type'] in ['dir', 'netfs']:
virt_pool = StoragePoolModel.get_storagepool(pool_name, self.conn)
virt_pool.refresh(0)
else:
def _stream_handler(stream, nbytes, fd):
return fd.read(nbytes)
virt_stream = virt_vol = None
try:
task = self.create(
pool_name, {
'name': name,
'format': 'raw',
'capacity': downloaded_size,
'allocation': downloaded_size
})
self.task.wait(task['id'])
virt_vol = StorageVolumeModel.get_storagevolume(
pool_name, name, self.conn)
virt_stream = self.conn.get().newStream(0)
virt_vol.upload(virt_stream, 0, downloaded_size, 0)
with open(file_path) as fd:
virt_stream.sendAll(_stream_handler, fd)
virt_stream.finish()
except (IOError, libvirt.libvirtError) as e:
try:
if virt_stream:
virt_stream.abort()
if virt_vol:
virt_vol.delete(0)
except libvirt.libvirtError, virt_e:
kimchi_log.error(virt_e.message)
finally:
raise OperationFailed('KCHVOL0007E', {
'name': name,
'pool': pool_name,
'err': e.message
})
finally:
class StorageVolumeModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.objstore = kargs['objstore']
self.task = TaskModel(**kargs)
self.storagevolumes = StorageVolumesModel(**kargs)
self.storagepool = StoragePoolModel(**kargs)
@staticmethod
def get_storagevolume(poolname, name, conn):
pool = StoragePoolModel.get_storagepool(poolname, conn)
if not pool.isActive():
raise InvalidOperation("KCHVOL0006E", {'name': pool})
try:
return pool.storageVolLookupByName(name.encode("utf-8"))
except libvirt.libvirtError as e:
if e.get_error_code() == libvirt.VIR_ERR_NO_STORAGE_VOL:
raise NotFoundError("KCHVOL0002E", {'name': name,
'pool': poolname})
else:
raise
def lookup(self, pool, name):
vol = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
path = vol.path()
info = vol.info()
xml = vol.XMLDesc(0)
try:
fmt = xpath_get_text(xml, "/volume/target/format/@type")[0]
except IndexError:
# Not all types of libvirt storage can provide volume format
# infomation. When there is no format information, we assume
# it's 'raw'.
fmt = 'raw'
iso_img = None
# 'raw' volumes from 'logical' pools may actually be 'iso';
# libvirt always reports them as 'raw'
pool_info = self.storagepool.lookup(pool)
if pool_info['type'] == 'logical' and fmt == 'raw':
try:
iso_img = IsoImage(path)
except IsoFormatError:
# not 'iso' afterall
pass
else:
fmt = 'iso'
ref_cnt = get_disk_ref_cnt(self.objstore, self.conn, path)
res = dict(type=VOLUME_TYPE_MAP[info[0]],
capacity=info[1],
allocation=info[2],
path=path,
ref_cnt=ref_cnt,
format=fmt)
if fmt == 'iso':
if os.path.islink(path):
path = os.path.join(os.path.dirname(path), os.readlink(path))
os_distro = os_version = 'unknown'
try:
if iso_img is None:
iso_img = IsoImage(path)
os_distro, os_version = iso_img.probe()
bootable = True
except IsoFormatError:
bootable = False
res.update(
dict(os_distro=os_distro, os_version=os_version, path=path,
bootable=bootable))
return res
def wipe(self, pool, name):
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
try:
volume.wipePattern(libvirt.VIR_STORAGE_VOL_WIPE_ALG_ZERO, 0)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0009E",
{'name': name, 'err': e.get_error_message()})
def delete(self, pool, name):
pool_info = StoragePoolModel(conn=self.conn,
objstore=self.objstore).lookup(pool)
if pool_info['type'] in READONLY_POOL_TYPE:
raise InvalidParameter("KCHVOL0012E", {'type': pool_info['type']})
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
try:
volume.delete(0)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0010E",
{'name': name, 'err': e.get_error_message()})
def resize(self, pool, name, size):
volume = StorageVolumeModel.get_storagevolume(pool, name, self.conn)
# When decreasing the storage volume capacity, the flag
# VIR_STORAGE_VOL_RESIZE_SHRINK must be used
flags = 0
if volume.info()[1] > size:
# FIXME: Even using VIR_STORAGE_VOL_RESIZE_SHRINK flag it is not
# possible to decrease the volume capacity due a libvirt bug
# For reference:
# - https://bugzilla.redhat.com/show_bug.cgi?id=1021802
flags = libvirt.VIR_STORAGE_VOL_RESIZE_SHRINK
try:
volume.resize(size, flags)
except libvirt.libvirtError as e:
raise OperationFailed("KCHVOL0011E",
{'name': name, 'err': e.get_error_message()})
def clone(self, pool, name, new_pool=None, new_name=None):
"""Clone a storage volume.
Arguments:
pool -- The name of the original pool.
name -- The name of the original volume.
new_pool -- The name of the destination pool (optional). If omitted,
the new volume will be created on the same pool as the
original one.
new_name -- The name of the new volume (optional). If omitted, a new
value based on the original volume's name will be used.
Return:
A Task running the clone operation.
"""
# the same pool will be used if no pool is specified
if new_pool is None:
new_pool = pool
# a default name based on the original name will be used if no name
# is specified
if new_name is None:
base, ext = os.path.splitext(name)
new_name = get_next_clone_name(self.storagevolumes.get_list(pool),
base, ext)
params = {'pool': pool,
'name': name,
'new_pool': new_pool,
'new_name': new_name}
taskid = add_task(u'/storagepools/%s/storagevolumes/%s' %
(pool, new_name), self._clone_task, self.objstore,
params)
return self.task.lookup(taskid)
def _clone_task(self, cb, params):
"""Asynchronous function which performs the clone operation.
This function copies all the data inside the original volume into the
new one.
Arguments:
cb -- A callback function to signal the Task's progress.
params -- A dict with the following values:
"pool": The name of the original pool.
"name": The name of the original volume.
"new_pool": The name of the destination pool.
"new_name": The name of the new volume.
"""
orig_pool_name = params['pool']
orig_vol_name = params['name']
new_pool_name = params['new_pool']
new_vol_name = params['new_name']
try:
cb('setting up volume cloning')
orig_vir_vol = StorageVolumeModel.get_storagevolume(orig_pool_name,
orig_vol_name,
self.conn)
orig_vol = self.lookup(orig_pool_name, orig_vol_name)
new_vir_pool = StoragePoolModel.get_storagepool(new_pool_name,
self.conn)
cb('building volume XML')
root_elem = E.volume()
root_elem.append(E.name(new_vol_name))
root_elem.append(E.capacity(unicode(orig_vol['capacity']),
unit='bytes'))
target_elem = E.target()
target_elem.append(E.format(type=orig_vol['format']))
root_elem.append(target_elem)
new_vol_xml = ET.tostring(root_elem, encoding='utf-8',
pretty_print=True)
cb('cloning volume')
new_vir_pool.createXMLFrom(new_vol_xml, orig_vir_vol, 0)
except (InvalidOperation, NotFoundError, libvirt.libvirtError), e:
raise OperationFailed('KCHVOL0023E',
{'name': orig_vol_name,
'pool': orig_pool_name,
'err': e.get_error_message()})
cb('adding volume to the object store')
new_vol_id = '%s:%s' % (new_pool_name, new_vol_name)
with self.objstore as session:
session.store('storagevolume', new_vol_id, {'ref_cnt': 0})
cb('OK', True)