def _do_deep_scan(self, params):
scan_params = dict(ignore_list=[])
scan_params['scan_path'] = params['path']
params['type'] = 'dir'
for pool in self.get_list():
try:
res = StoragePoolModel(conn=self.conn,
objstore=self.objstore).lookup(pool)
if res['state'] == 'active':
scan_params['ignore_list'].append(res['path'])
except Exception as e:
wok_log.debug(f'Exception {e} occured when get ignore path')
params['path'] = self.scanner.scan_dir_prepare(params['name'])
scan_params['pool_path'] = params['path']
task_id = AsyncTask(
f'/plugins/kimchi/storagepools/{ISO_POOL_NAME}',
self.scanner.start_scan,
scan_params,
).id
# Record scanning-task/storagepool mapping for future querying
try:
with self.objstore as session:
session.store('scanning', params['name'], task_id,
get_kimchi_version())
return task_id
except Exception as e:
raise OperationFailed('KCHPOOL0037E', {'err': e.message})
def get_disk_used_by(objstore, conn, path):
try:
with objstore as session:
try:
used_by = session.get("storagevolume", path)["used_by"]
except (KeyError, NotFoundError):
wok_log.info("Volume %s not found in obj store." % path)
used_by = []
# try to find this volume in existing vm
vms_list = VMsModel.get_vms(conn)
for vm in vms_list:
dom = VMModel.get_vm(vm, conn)
storages = get_vm_disks(dom)
for disk in storages.keys():
d_info = get_vm_disk_info(dom, disk)
if path == d_info["path"]:
used_by.append(vm)
try:
session.store("storagevolume", path, {"used_by": used_by}, get_kimchi_version())
except Exception as e:
# Let the exception be raised. If we allow disks'
# used_by to be out of sync, data corruption could
# occour if a disk is added to two guests
# unknowingly.
wok_log.error("Unable to store storage volume id in" " objectstore due error: %s", e.message)
raise OperationFailed("KCHVOL0017E", {"err": e.message})
except Exception as e:
# This exception is going to catch errors returned by 'with',
# specially ones generated by 'session.store'. It is outside
# to avoid conflict with the __exit__ function of 'with'
raise OperationFailed("KCHVOL0017E", {"err": e.message})
return used_by
def upgrade_objectstore_data(item, old_uri, new_uri):
"""
Upgrade the value of a given JSON's item of all Template and VM entries
of the objectstore from old_uri to new_uri.
"""
total = 0
try:
conn = sqlite3.connect(config.get_object_store(), timeout=10)
cursor = conn.cursor()
sql = "SELECT id, json FROM objects WHERE type='template' OR type='vm'"
cursor.execute(sql)
for row in cursor.fetchall():
# execute update here
template = json.loads(row[1])
path = template[item] if item in template else 'none'
if path.startswith(old_uri):
template[item] = new_uri + path
sql = 'UPDATE objects SET json=?, version=? WHERE id=?'
cursor.execute(
sql, (json.dumps(template),
config.get_kimchi_version(), row[0])
)
conn.commit()
total += 1
except sqlite3.Error as e:
if conn:
conn.rollback()
wok_log.error('Error while upgrading objectstore data: %s', e.args[0])
raise OperationFailed('KCHUTILS0006E')
finally:
if conn:
conn.close()
wok_log.info("%d '%s' entries upgraded in objectstore.", total, item)
def upgrade_objectstore_data(item, old_uri, new_uri):
"""
Upgrade the value of a given JSON's item of all Template and VM entries
of the objectstore from old_uri to new_uri.
"""
total = 0
try:
conn = sqlite3.connect(config.get_object_store(), timeout=10)
cursor = conn.cursor()
sql = "SELECT id, json FROM objects WHERE type='template' OR type='vm'"
cursor.execute(sql)
for row in cursor.fetchall():
# execute update here
template = json.loads(row[1])
path = (template[item] if item in template else 'none')
if path.startswith(old_uri):
template[item] = new_uri + path
sql = "UPDATE objects SET json=?, version=? WHERE id=?"
cursor.execute(sql, (json.dumps(template),
config.get_kimchi_version(), row[0]))
conn.commit()
total += 1
except sqlite3.Error, e:
if conn:
conn.rollback()
wok_log.error("Error while upgrading objectstore data: %s", e.args[0])
raise OperationFailed("KCHUTILS0006E")
def get_disk_used_by(objstore, conn, path):
try:
with objstore as session:
try:
used_by = session.get('storagevolume', path)['used_by']
except (KeyError, NotFoundError):
wok_log.info('Volume %s not found in obj store.' % path)
used_by = []
# try to find this volume in existing vm
vms_list = VMsModel.get_vms(conn)
for vm in vms_list:
dom = VMModel.get_vm(vm, conn)
storages = get_vm_disks(dom)
for disk in storages.keys():
d_info = get_vm_disk_info(dom, disk)
if path == d_info['path']:
used_by.append(vm)
try:
session.store('storagevolume', path, {'used_by': used_by},
get_kimchi_version())
except Exception as e:
# Let the exception be raised. If we allow disks'
# used_by to be out of sync, data corruption could
# occour if a disk is added to two guests
# unknowingly.
wok_log.error(
'Unable to store storage volume id in'
' objectstore due error: %s', e.message)
raise OperationFailed('KCHVOL0017E', {'err': e.message})
except Exception as e:
# This exception is going to catch errors returned by 'with',
# specially ones generated by 'session.store'. It is outside
# to avoid conflict with the __exit__ function of 'with'
raise OperationFailed('KCHVOL0017E', {'err': e.message})
return used_by
def set_disk_used_by(objstore, path, new_used_by):
try:
with objstore as session:
session.store('storagevolume', path, {'used_by': new_used_by},
get_kimchi_version())
except Exception as e:
raise OperationFailed('KCHVOL0017E', {'err': e.message})
def save_template(self, params):
# Creates the template class with necessary information
t = LibvirtVMTemplate(params, scan=True, conn=self.conn)
# Validate cpu info
t.cpuinfo_validate()
# Validate memory
t._validate_memory()
# Validate volumes
for disk in t.info.get('disks'):
volume = disk.get('volume')
# volume can be None
if 'volume' in disk.keys():
self.template_volume_validate(volume, disk['pool'])
# template with the same name already exists: raise exception
name = params['name']
with self.objstore as session:
if name in session.get_list('template'):
raise InvalidOperation("KCHTMPL0001E", {'name': name})
# Store template on objectstore
try:
with self.objstore as session:
session.store('template', name, t.info,
get_kimchi_version())
except InvalidOperation:
raise
except Exception, e:
raise OperationFailed('KCHTMPL0020E', {'err': e.message})
def set_disk_used_by(objstore, path, new_used_by):
try:
with objstore as session:
session.store('storagevolume', path, {'used_by': new_used_by},
get_kimchi_version())
except Exception as e:
raise OperationFailed('KCHVOL0017E', {'err': e.message})
def _do_deep_scan(self, params):
scan_params = dict(ignore_list=[])
scan_params['scan_path'] = params['path']
params['type'] = 'dir'
for pool in self.get_list():
try:
res = StoragePoolModel(conn=self.conn, objstore=self.objstore).lookup(
pool
)
if res['state'] == 'active':
scan_params['ignore_list'].append(res['path'])
except Exception as e:
wok_log.debug(f'Exception {e} occured when get ignore path')
params['path'] = self.scanner.scan_dir_prepare(params['name'])
scan_params['pool_path'] = params['path']
task_id = AsyncTask(
f'/plugins/kimchi/storagepools/{ISO_POOL_NAME}',
self.scanner.start_scan,
scan_params,
).id
# Record scanning-task/storagepool mapping for future querying
try:
with self.objstore as session:
session.store(
'scanning', params['name'], task_id, get_kimchi_version())
return task_id
except Exception as e:
raise OperationFailed('KCHPOOL0037E', {'err': e.message})
def create(self, params):
name = params.get('name', '').strip()
iso = params.get('cdrom')
# check search permission
if iso and iso.startswith('/') and os.path.exists(iso):
st_mode = os.stat(iso).st_mode
if stat.S_ISREG(st_mode) or stat.S_ISBLK(st_mode):
user = UserTests().probe_user()
run_setfacl_set_attr(iso, user=user)
ret, excp = probe_file_permission_as_user(iso, user)
if ret is False:
raise InvalidParameter('KCHISO0008E', {
'filename': iso,
'user': user,
'err': excp
})
conn = self.conn.get()
for net_name in params.get(u'networks', []):
try:
conn.networkLookupByName(net_name.encode('utf-8'))
except Exception:
raise InvalidParameter("KCHTMPL0003E", {
'network': net_name,
'template': name
})
# Creates the template class with necessary information
# Checkings will be done while creating this class, so any exception
# will be raised here
t = LibvirtVMTemplate(params, scan=True, conn=self.conn)
# Validate cpu info
t.cpuinfo_validate()
# Validate memory
t._validate_memory()
# Validate volumes
for disk in t.info.get('disks'):
volume = disk.get('volume')
# volume can be None
if 'volume' in disk.keys():
self.template_volume_validate(volume, disk['pool'])
# Store template on objectstore
name = params['name']
try:
with self.objstore as session:
if name in session.get_list('template'):
raise InvalidOperation("KCHTMPL0001E", {'name': name})
session.store('template', name, t.info, get_kimchi_version())
except InvalidOperation:
raise
except Exception, e:
raise OperationFailed('KCHTMPL0020E', {'err': e.message})
def create(self, params):
name = params.get('name', '').strip()
iso = params.get('cdrom')
# check search permission
if iso and iso.startswith('/') and os.path.exists(iso):
st_mode = os.stat(iso).st_mode
if stat.S_ISREG(st_mode) or stat.S_ISBLK(st_mode):
user = UserTests().probe_user()
run_setfacl_set_attr(iso, user=user)
ret, excp = probe_file_permission_as_user(iso, user)
if ret is False:
raise InvalidParameter('KCHISO0008E',
{'filename': iso, 'user': user,
'err': excp})
conn = self.conn.get()
for net_name in params.get(u'networks', []):
try:
conn.networkLookupByName(net_name.encode('utf-8'))
except Exception:
raise InvalidParameter("KCHTMPL0003E", {'network': net_name,
'template': name})
# Creates the template class with necessary information
# Checkings will be done while creating this class, so any exception
# will be raised here
t = LibvirtVMTemplate(params, scan=True, conn=self.conn)
# Validate cpu info
t.cpuinfo_validate()
# Validate max memory
maxMem = (t._get_max_memory(t.info.get('memory')) >> 10)
if t.info.get('memory') > maxMem:
raise OperationFailed("KCHVM0041E", {'maxmem': str(maxMem)})
# Validate volumes
for disk in t.info.get('disks'):
volume = disk.get('volume')
# volume can be None
if 'volume' in disk.keys():
self.template_volume_validate(volume, disk['pool'])
# Store template on objectstore
name = params['name']
try:
with self.objstore as session:
if name in session.get_list('template'):
raise InvalidOperation("KCHTMPL0001E", {'name': name})
session.store('template', name, t.info,
get_kimchi_version())
except InvalidOperation:
raise
except Exception, e:
raise OperationFailed('KCHTMPL0020E', {'err': e.message})
def test_image_based_template(self):
inst = model.Model(objstore_loc=self.tmp_store)
with RollbackContext() as rollback:
vol = "base-vol.img"
params = {"name": vol, "capacity": 1073741824, "allocation": 1048576, "format": "qcow2"} # 1 GiB # 1 MiB
task_id = inst.storagevolumes_create("default", params)["id"]
rollback.prependDefer(inst.storagevolume_delete, "default", vol)
inst.task_wait(task_id)
self.assertEquals("finished", inst.task_lookup(task_id)["status"])
vol_path = inst.storagevolume_lookup("default", vol)["path"]
# Hack the model objstore to add a new template
# It is needed as the image file must be a bootable image when
# using model
# As it is difficult to create one on test runtime, inject a
# template with an empty image file to the objstore to test the
# feature
tmpl_name = "img-tmpl"
tmpl_info = {
"cpus": 1,
"cdrom": "",
"graphics": {"type": "vnc", "listen": "127.0.0.1"},
"networks": ["default"],
"memory": 1024,
"folder": [],
"icon": "images/icon-vm.png",
"os_distro": "unknown",
"os_version": "unknown",
"disks": [{"base": vol_path, "size": 10}],
"storagepool": "/plugins/kimchi/storagepools/default",
}
with inst.objstore as session:
session.store("template", tmpl_name, tmpl_info, get_kimchi_version())
params = {"name": "kimchi-vm", "template": "/plugins/kimchi/templates/img-tmpl"}
task = inst.vms_create(params)
inst.task_wait(task["id"])
rollback.prependDefer(inst.vm_delete, "kimchi-vm")
vms = inst.vms_get_list()
self.assertTrue("kimchi-vm" in vms)
inst.vm_start("kimchi-vm")
rollback.prependDefer(inst.vm_poweroff, "kimchi-vm")
info = inst.vm_lookup("kimchi-vm")
self.assertEquals("running", info["state"])
def test_image_based_template(self):
inst = model.Model(objstore_loc=self.tmp_store)
with RollbackContext() as rollback:
vol = 'base-vol.img'
params = {'name': vol,
'capacity': 1073741824, # 1 GiB
'allocation': 1048576, # 1 MiB
'format': 'qcow2'}
task_id = inst.storagevolumes_create('default', params)['id']
rollback.prependDefer(inst.storagevolume_delete, 'default', vol)
inst.task_wait(task_id)
self.assertEquals('finished', inst.task_lookup(task_id)['status'])
vol_path = inst.storagevolume_lookup('default', vol)['path']
# Hack the model objstore to add a new template
# It is needed as the image file must be a bootable image when
# using model
# As it is difficult to create one on test runtime, inject a
# template with an empty image file to the objstore to test the
# feature
tmpl_name = "img-tmpl"
tmpl_info = {"cpus": 1, "cdrom": "",
"graphics": {"type": "vnc", "listen": "127.0.0.1"},
"networks": ["default"], "memory": 1024, "folder": [],
"icon": "images/icon-vm.png",
"os_distro": "unknown", "os_version": "unknown",
"disks": [{"base": vol_path, "size": 10}],
"storagepool": "/plugins/kimchi/storagepools/default"}
with inst.objstore as session:
session.store('template', tmpl_name, tmpl_info,
get_kimchi_version())
params = {'name': 'kimchi-vm',
'template': '/plugins/kimchi/templates/img-tmpl'}
task = inst.vms_create(params)
inst.task_wait(task['id'])
rollback.prependDefer(inst.vm_delete, 'kimchi-vm')
vms = inst.vms_get_list()
self.assertTrue('kimchi-vm' in vms)
inst.vm_start('kimchi-vm')
rollback.prependDefer(inst.vm_poweroff, 'kimchi-vm')
info = inst.vm_lookup('kimchi-vm')
self.assertEquals('running', info['state'])
def main():
repo, distros, user, password = usage()
kimchi_version = get_kimchi_version()
wok_version = get_version()
for distro in distros:
distro_name = distro.split("/")
if distro_name[0] == 'ubuntu':
pm = 'debian'
else:
pm = distro_name[0]
try:
shutil.rmtree(HOMEWOK)
except:
pass
execute_cmd([COMMANDS_OS[pm]['update']], 'Updating system')
execute_cmd(PACKAGES['wok'], 'Cloning Wok')
execute_cmd(PACKAGES['kimchi'], 'Cloning Kimchi')
install_dependencies(distro_name[0], pm)
execute_cmd([COMMANDS_OS[pm]['pip']],'Installing Pip packages')
for item in BUILD:
run_build(item, HOMEWOK)
run_build(item, HOMEKIMCHI)
run_build(COMMANDS_OS[pm]['make'], HOMEWOK)
run_build(COMMANDS_OS[pm]['make'], HOMEKIMCHI)
wok_package = 'wok-' + wok_version + '.' + distro_name[0] + '.noarch' + COMMANDS_OS[pm]['pk']
kimchi_package = 'kimchi-' + kimchi_version + '.noarch' + COMMANDS_OS[pm]['pk']
curl_cmd(repo, distro_name[0], distro, wok_package, user, password, HOMEWOK + wok_package, 'wok')
curl_cmd(repo, distro_name[0], distro, kimchi_package, user, password, HOMEKIMCHI + kimchi_package, 'kimchi')
print("All Good, check JFROG")
def lookup(self, name):
kconfig = config.get('kimchi', {})
return {
'federation': kconfig.get('federation', False),
'version': get_kimchi_version()
}
def create(self, params):
name = params.get('name', '').strip()
iso = params.get('cdrom')
# check search permission
if iso and iso.startswith('/') and os.path.exists(iso):
st_mode = os.stat(iso).st_mode
if stat.S_ISREG(st_mode) or stat.S_ISBLK(st_mode):
user = UserTests().probe_user()
run_setfacl_set_attr(iso, user=user)
ret, excp = probe_file_permission_as_user(iso, user)
if ret is False:
raise InvalidParameter('KCHISO0008E',
{'filename': iso, 'user': user,
'err': excp})
cpu_info = params.get('cpu_info')
if cpu_info:
topology = cpu_info.get('topology')
# Check, even though currently only topology
# is supported.
if topology:
sockets = topology['sockets']
cores = topology['cores']
threads = topology['threads']
if params.get('cpus') is None:
params['cpus'] = sockets * cores * threads
# check_topoology will raise the appropriate
# exception if a topology is invalid.
CPUInfoModel(conn=self.conn).\
check_topology(params['cpus'], topology)
conn = self.conn.get()
for net_name in params.get(u'networks', []):
try:
conn.networkLookupByName(net_name.encode('utf-8'))
except Exception:
raise InvalidParameter("KCHTMPL0003E", {'network': net_name,
'template': name})
# Creates the template class with necessary information
# Checkings will be done while creating this class, so any exception
# will be raised here
t = LibvirtVMTemplate(params, scan=True, conn=self.conn)
# Validate volumes
for disk in t.info.get('disks'):
volume = disk.get('volume')
# volume can be None
if 'volume' in disk.keys():
self.template_volume_validate(volume, disk['pool'])
# Store template on objectstore
name = params['name']
try:
with self.objstore as session:
if name in session.get_list('template'):
raise InvalidOperation("KCHTMPL0001E", {'name': name})
session.store('template', name, t.info,
get_kimchi_version())
except InvalidOperation:
raise
except Exception, e:
raise OperationFailed('KCHTMPL0020E', {'err': e.message})
def lookup(self, name):
kconfig = config.get("kimchi", {})
return {"federation": kconfig.get("federation", False), "version": get_kimchi_version()}
objstore=self.objstore).lookup(pool)
if res['state'] == 'active':
scan_params['ignore_list'].append(res['path'])
except Exception, e:
err = "Exception %s occured when get ignore path"
wok_log.debug(err % e.message)
params['path'] = self.scanner.scan_dir_prepare(params['name'])
scan_params['pool_path'] = params['path']
task_id = add_task('/plugins/kimchi/storagepools/%s' % ISO_POOL_NAME,
self.scanner.start_scan, self.objstore, scan_params)
# Record scanning-task/storagepool mapping for future querying
try:
with self.objstore as session:
session.store('scanning', params['name'], task_id,
get_kimchi_version())
return task_id
except Exception as e:
raise OperationFailed('KCHPOOL0037E', {'err': e.message})
class StoragePoolModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.objstore = kargs['objstore']
@staticmethod
def get_storagepool(name, conn):
conn = conn.get()
try:
return conn.storagePoolLookupByName(name.encode("utf-8"))
def lookup(self, name):
kconfig = config.get('kimchi', {})
return {'federation': kconfig.get('federation', False),
'version': get_kimchi_version()}
def lookup(self, name):
return {'version': get_kimchi_version()}
def lookup(self, name):
return {'version': get_kimchi_version()}
objstore=self.objstore).lookup(pool)
if res['state'] == 'active':
scan_params['ignore_list'].append(res['path'])
except Exception, e:
err = "Exception %s occured when get ignore path"
wok_log.debug(err % e.message)
params['path'] = self.scanner.scan_dir_prepare(params['name'])
scan_params['pool_path'] = params['path']
task_id = AsyncTask('/plugins/kimchi/storagepools/%s' % ISO_POOL_NAME,
self.scanner.start_scan, scan_params).id
# Record scanning-task/storagepool mapping for future querying
try:
with self.objstore as session:
session.store('scanning', params['name'], task_id,
get_kimchi_version())
return task_id
except Exception as e:
raise OperationFailed('KCHPOOL0037E', {'err': e.message})
class StoragePoolModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
self.objstore = kargs['objstore']
@staticmethod
def get_storagepool(name, conn):
conn = conn.get()
try:
return conn.storagePoolLookupByName(name.encode("utf-8"))
def set_disk_used_by(objstore, path, new_used_by):
try:
with objstore as session:
session.store("storagevolume", path, {"used_by": new_used_by}, get_kimchi_version())
except Exception as e:
raise OperationFailed("KCHVOL0017E", {"err": e.message})
def lookup(self, name):
return {
'version': get_kimchi_version(),
'with_spice_web_client': with_spice_web_client()
}
def create(self, params):
name = params.get('name', '').strip()
iso = params.get('cdrom')
# check search permission
if iso and iso.startswith('/') and os.path.exists(iso):
st_mode = os.stat(iso).st_mode
if stat.S_ISREG(st_mode) or stat.S_ISBLK(st_mode):
user = UserTests().probe_user()
run_setfacl_set_attr(iso, user=user)
ret, excp = probe_file_permission_as_user(iso, user)
if ret is False:
raise InvalidParameter('KCHISO0008E', {
'filename': iso,
'user': user,
'err': excp
})
cpu_info = params.get('cpu_info')
if cpu_info:
topology = cpu_info.get('topology')
# Check, even though currently only topology
# is supported.
if topology:
sockets = topology['sockets']
cores = topology['cores']
threads = topology['threads']
if params.get('cpus') is None:
params['cpus'] = sockets * cores * threads
# check_topoology will raise the appropriate
# exception if a topology is invalid.
CPUInfoModel(conn=self.conn).\
check_topology(params['cpus'], topology)
conn = self.conn.get()
pool_uri = params.get(u'storagepool', '')
if pool_uri:
try:
pool_name = pool_name_from_uri(pool_uri)
pool = conn.storagePoolLookupByName(pool_name.encode("utf-8"))
except Exception:
raise InvalidParameter("KCHTMPL0004E", {
'pool': pool_uri,
'template': name
})
tmp_volumes = [
disk['volume'] for disk in params.get('disks', [])
if 'volume' in disk
]
self.template_volume_validate(tmp_volumes, pool)
for net_name in params.get(u'networks', []):
try:
conn.networkLookupByName(net_name.encode('utf-8'))
except Exception:
raise InvalidParameter("KCHTMPL0003E", {
'network': net_name,
'template': name
})
# Creates the template class with necessary information
# Checkings will be done while creating this class, so any exception
# will be raised here
t = LibvirtVMTemplate(params, scan=True, conn=self.conn)
name = params['name']
try:
with self.objstore as session:
if name in session.get_list('template'):
raise InvalidOperation("KCHTMPL0001E", {'name': name})
session.store('template', name, t.info, get_kimchi_version())
except InvalidOperation:
raise
except Exception, e:
raise OperationFailed('KCHTMPL0020E', {'err': e.message})