def test_optional_value(self):
schema = Schema({"key": str, Optional("op_key"): IntVal()})
data = schema.validate({"key": "abc"})
self.assertEqual({"key": "abc"}, data)
data = schema.validate({'key': 'abc', 'op_key': 123})
self.assertEqual({'key': 'abc', 'op_key': 123}, data)
with self.assertRaises(SchemaError):
schema.validate({'key': 'abc', 'op_key': 'bcd'})
def test_donot_care(self):
schema = Schema({
'key': str,
DoNotCare(str): object
})
data = schema.validate({'key': 'abc', 'key2': 'bbb', 'key3': [1, 2, 3]})
self.assertEqual({'key': 'abc', 'key2': 'bbb', 'key3': [1, 2, 3]}, data)
with self.assertRaises(SchemaError):
schema.validate({'key2': 'bbb', 'key3': [1, 2, 3]})
def test_optional_value(self):
schema = Schema({
"key": str,
Optional("op_key"): IntVal()
})
data = schema.validate({"key": "abc"})
self.assertEqual({"key": "abc"}, data)
data = schema.validate({'key': 'abc', 'op_key': 123})
self.assertEqual({'key': 'abc', 'op_key': 123}, data)
with self.assertRaises(SchemaError):
schema.validate({'key': 'abc', 'op_key': 'bcd'})
def test_default_value(self):
schema = Schema({
"key": str,
Optional('op_key'): Default(IntVal(min=10), default=50)
})
data = schema.validate({'key': 'abc'})
self.assertEqual({'key': 'abc', 'op_key': 50}, data)
data = schema.validate({'key': 'abc', 'op_key': 20})
self.assertEqual({'key': 'abc', 'op_key': 20}, data)
with self.assertRaises(SchemaError):
schema.validate({'key': 'abc', 'op_key': 0})
def test_default_value(self):
schema = Schema({
"key": str,
Optional('op_key'): Default(IntVal(min=10), default=50)
})
data = schema.validate({'key': 'abc'})
self.assertEqual({'key': 'abc', 'op_key': 50}, data)
data = schema.validate({'key': 'abc', 'op_key': 20})
self.assertEqual({'key': 'abc', 'op_key': 20}, data)
with self.assertRaises(SchemaError):
schema.validate({'key': 'abc', 'op_key': 0})
def test_donot_care(self):
schema = Schema({'key': str, DoNotCare(str): object})
data = schema.validate({
'key': 'abc',
'key2': 'bbb',
'key3': [1, 2, 3]
})
self.assertEqual({
'key': 'abc',
'key2': 'bbb',
'key3': [1, 2, 3]
}, data)
with self.assertRaises(SchemaError):
schema.validate({'key2': 'bbb', 'key3': [1, 2, 3]})
def test_autodel(self):
schema = Schema({'key': str, AutoDel(str): object})
schema2 = Schema({'key': str})
data = schema.validate({
'key': 'abc',
'key2': 'bbb',
'key3': [1, 2, 3]
})
self.assertEqual({'key': 'abc'}, data)
with self.assertRaises(SchemaError):
schema2.validate({'key': 'abc', 'key2': 'bbb', 'key3': [1, 2, 3]})
with self.assertRaises(SchemaError):
schema.validate({'key2': 'bbb', 1: [1, 2, 3]})
def test_autodel(self):
schema = Schema({
'key': str,
AutoDel(str): object
})
schema2 = Schema({
'key': str
})
data = schema.validate({'key': 'abc', 'key2': 'bbb', 'key3': [1, 2, 3]})
self.assertEqual({'key': 'abc'}, data)
with self.assertRaises(SchemaError):
schema2.validate({'key': 'abc', 'key2': 'bbb', 'key3': [1, 2, 3]})
with self.assertRaises(SchemaError):
schema.validate({'key2': 'bbb', 1: [1, 2, 3]})
def test_bool_value(self):
schema = Schema(BoolVal())
self.assertEquals(True, schema.validate(True))
self.assertEquals(True, schema.validate("True"))
self.assertEquals(True, schema.validate("true"))
self.assertEquals(False, schema.validate(False))
self.assertEquals(False, schema.validate("False"))
self.assertEquals(False, schema.validate("false"))
with self.assertRaises(SchemaError):
schema.validate(0)
with self.assertRaises(SchemaError):
schema.validate(1)
with self.assertRaises(SchemaError):
schema.validate("abc")
def test_bool_value(self):
schema = Schema(BoolVal())
self.assertEquals(True, schema.validate(True))
self.assertEquals(True, schema.validate("True"))
self.assertEquals(True, schema.validate("true"))
self.assertEquals(False, schema.validate(False))
self.assertEquals(False, schema.validate("False"))
self.assertEquals(False, schema.validate("false"))
with self.assertRaises(SchemaError):
schema.validate(0)
with self.assertRaises(SchemaError):
schema.validate(1)
with self.assertRaises(SchemaError):
schema.validate("abc")
def test_strre_value(self):
schema = Schema(StrRe("^(abc|efg)$"))
self.assertEquals("abc", schema.validate("abc"))
self.assertEquals("efg", schema.validate("efg"))
with self.assertRaises(SchemaError):
schema.validate("ebc")
with self.assertRaises(SchemaError):
schema.validate("abcdefg")
with self.assertRaises(SchemaError):
schema.validate(0)
with self.assertRaises(SchemaError):
schema.validate(1)
def test_strre_value(self):
schema = Schema(StrRe("^(abc|efg)$"))
self.assertEquals("abc", schema.validate("abc"))
self.assertEquals("efg", schema.validate("efg"))
with self.assertRaises(SchemaError):
schema.validate("ebc")
with self.assertRaises(SchemaError):
schema.validate("abcdefg")
with self.assertRaises(SchemaError):
schema.validate(0)
with self.assertRaises(SchemaError):
schema.validate(1)
def test_list(self):
schema = Schema([str])
self.assertEqual(schema.validate(['abc', 'bbc', 'ddc']),
['abc', 'bbc', 'ddc'])
with self.assertRaises(SchemaError):
schema.validate(['abc', 123, 'bbc'])
schema = Schema([IntVal(min=10, max=20)])
self.assertEqual(schema.validate([10, 12, 19, 11]), [10, 12, 19, 11])
with self.assertRaises(SchemaError):
schema.validate([10, 12, 21])
def test_list(self):
schema = Schema([str])
self.assertEqual(schema.validate(['abc', 'bbc', 'ddc']),
['abc', 'bbc', 'ddc'])
with self.assertRaises(SchemaError):
schema.validate(['abc', 123, 'bbc'])
schema = Schema([IntVal(min=10, max=20)])
self.assertEqual(schema.validate([10, 12, 19, 11]), [10, 12, 19, 11])
with self.assertRaises(SchemaError):
schema.validate([10, 12, 21])
class FileSystemManager(object):
"""contains all methods to manage ethernet interface in linux system"""
def __init__(self):
# need a mutex to protect create/delete bond interface
self.lock = lock()
self.support_fs_type = {}
self.conf_file = os.path.join(STORLEVER_CONF_DIR, FS_CONF_FILE_NAME)
self.fs_conf_schema = Schema({
"type": Use(str), # filesystem type
"dev_file": Use(str), # dev file
"dev_uuid": Use(str), # dev uuid
"mount_point": Use(str), # mount point of this fs,
"mount_option": Use(str), # mount option of this fs
"check_onboot": BoolVal(), # fsck fs on boot
Optional("comment"): Default(Use(str), default=""), # comment,
AutoDel(str): object # for all other key we auto delete
})
self.fs_dict_schema = Schema({
DoNotCare(str): self.fs_conf_schema
})
# sync fs conf to fstab on boot
self.sync_to_fstab()
def _uuid_to_dev_file(self, uuid):
try:
return check_output(["/sbin/blkid", "-U", uuid]).strip()
except Exception:
return ""
def _dev_file_to_uuid(self, dev_file):
try:
return subprocess.check_output(
["/sbin/blkid", "-s", "UUID", "-o", "value", dev_file],
stderr=subprocess.STDOUT,
shell=False).strip()
except subprocess.CalledProcessError as e:
if e.returncode == 2:
http_status = 400
info = "The dev file (%s) has no UUID tag. " \
"Make sure it exists and contains a filesystem" % dev_file
else:
http_status = 500
info = e.output
# re-raise the storlever's error
raise StorLeverError(info, http_status)
def _load_conf(self):
fs_dict = {}
cfg_mgr().check_conf_dir()
if os.path.exists(self.conf_file):
fs_dict = \
Config.from_file(self.conf_file, self.fs_dict_schema).conf
# check dev_file by uuid
for fs_name, fs_conf in fs_dict.items():
if fs_conf["dev_uuid"] != "":
fs_conf["dev_file"] = self._uuid_to_dev_file(fs_conf["dev_uuid"])
return fs_dict
def _save_conf(self, fs_dict):
cfg_mgr().check_conf_dir()
Config.to_file(self.conf_file, fs_dict)
def _fs_conf_to_fstab_line(self, fs_name, fs_conf):
if fs_conf["dev_uuid"] == "":
dev_file_name = fs_conf["dev_file"]
else:
dev_file_name = "UUID=%s" % fs_conf["dev_uuid"]
if fs_conf["check_onboot"]:
boot_flag = 2
else:
boot_flag = 0
# get a fs object
fs_object = self._get_fs_type_cls(fs_conf["type"])(fs_name, fs_conf)
if fs_object.mount_options == "":
option_flag = "defaults"
else:
option_flag = fs_object.mount_options
return "%s\t%s\t%s\t%s\t0\t%d\n" % \
(dev_file_name, fs_conf["mount_point"], fs_conf["type"],
option_flag, boot_flag)
def _sync_to_fstab(self, fs_dict):
if os.path.exists(FSTAB_FILE_PATH):
with open(FSTAB_FILE_PATH, "r") as f:
lines = f.readlines()
else:
lines = []
if "# begin storlever\n" in lines:
before_storlever = lines[0:lines.index("# begin storlever\n")]
else:
before_storlever = lines[0:]
if before_storlever and (not before_storlever[-1].endswith("\n")):
before_storlever[-1] += "\n"
if "# end storlever\n" in lines:
after_storlever = lines[lines.index("# end storlever\n") + 1:]
else:
after_storlever = []
with open(FSTAB_FILE_PATH, "w") as f:
f.writelines(before_storlever)
f.write("# begin storlever\n")
for fs_name, fs_conf in fs_dict.items():
f.write(self._fs_conf_to_fstab_line(fs_name, fs_conf))
f.write("# end storlever\n")
f.writelines(after_storlever)
def sync_to_fstab(self):
"""sync the fs conf list in storlever to /etc/fstab"""
with self.lock:
fs_dict = self._load_conf()
self._sync_to_fstab(fs_dict)
def system_restore_cb(self):
"""sync the fs conf list in storlever to /etc/fstab"""
with self.lock:
fs_dict = {}
self._sync_to_fstab(fs_dict)
def _mount_fs(self, name, fs_conf):
# get a fs object
fs_object = self._get_fs_type_cls(fs_conf["type"])(name, fs_conf)
# call this object's mount method
fs_object.mount()
def _umount_fs(self, name, fs_conf):
# get a fs object
fs_object = self._get_fs_type_cls(fs_conf["type"])(name, fs_conf)
# call this object's mount method
fs_object.umount()
def _get_fs_type_cls(self, type):
cls = self.support_fs_type.get(type, fs.FileSystem)
return cls
def add_fs_type(self, type, cls, *args, **kwargs):
"""add the fs class with specific type name"""
with self.lock:
self.support_fs_type[type] = cls
def get_fs_by_name(self, fs_name):
"""return a fs object according to the given fs name"""
with self.lock:
fs_dict = self._load_conf()
if fs_name not in fs_dict:
raise StorLeverError("Filesystem(%s) does not exist" % fs_name, 404)
fs_conf = fs_dict[fs_name]
cls = self._get_fs_type_cls(fs_conf["type"])
return cls(fs_name, fs_conf)
def get_fs_list(self):
"""get the fs object list in the storlever
"""
with self.lock:
fs_dict = self._load_conf()
fs_list = []
for fs_name, fs_conf in fs_dict.items():
cls = self._get_fs_type_cls(fs_conf["type"])
fs_list.append(cls(fs_name, fs_conf))
return fs_list
def fs_type_list(self):
"""list all fs type supported in the storlever"""
with self.lock:
type_list = self.support_fs_type.keys()
return type_list
def add_fs(self, fs_name, type, dev_file,
mount_option="", check_onboot=False,
comment="", user="******"):
"""add a filesystem with the given properties to storlever
The new filesystem would be mount on the specific directory(/mnt/FS_NAME)
and would be added to the storlever's fs config
"""
# check type
if type not in self.support_fs_type:
raise StorLeverError("type(%s) does not support" % type, 400)
# check mount point
mount_point = os.path.join(MOUNT_DIR, fs_name)
if os.path.exists(mount_point):
if not os.path.isdir(mount_point):
raise StorLeverError("mount point(%s) already exists and is not directory" % mount_point)
else:
# create mount point
os.makedirs(mount_point)
# don't check dev file exist, because for the network fs, the dev file is a network id
# if not os.path.exists(dev_file):
# raise StorLeverError("dev file(%s) does not exist" % dev_file, 400)
dev_uuid = ""
if (not dev_file.startswith("/dev/mapper")) and os.path.exists(dev_file):
dev_uuid = self._dev_file_to_uuid(dev_file)
fs_conf = {
"type": type,
"dev_file": dev_file,
"dev_uuid": dev_uuid,
"mount_point": mount_point,
"mount_option": mount_option,
"check_onboot": check_onboot,
"comment": comment
}
fs_conf = self.fs_conf_schema.validate(fs_conf)
with self.lock:
fs_dict = self._load_conf()
if fs_name in fs_dict:
raise StorLeverError("filesystem(%s) already exist" % fs_name, 400)
# mount fs first
self._mount_fs(fs_name, fs_conf)
fs_dict[fs_name] = fs_conf
self._save_conf(fs_dict)
self._sync_to_fstab(fs_dict)
logger.log(logging.INFO, logger.LOG_TYPE_CONFIG,
"New filesystem %s (dev:%s, mount_point:%s, option:%s) "
"is added by user(%s)" %
(fs_name, dev_file, mount_point, mount_option, user))
def del_fs(self, fs_name, user="******"):
"""delete a filesystem from storlever
the file would be deleted from the storlever's config file and
would be unmount from linux system
"""
with self.lock:
fs_dict = self._load_conf()
if fs_name not in fs_dict:
raise StorLeverError("filesystem(%s) does not exist" % fs_name, 400)
fs_conf = fs_dict[fs_name]
del fs_dict[fs_name]
#umount fs first. if it failed, don't delete it in the config
self._umount_fs(fs_name, fs_conf)
self._save_conf(fs_dict)
self._sync_to_fstab(fs_dict)
try:
os.rmdir(fs_conf["mount_point"])
except OSError as e:
pass
logger.log(logging.INFO, logger.LOG_TYPE_CONFIG,
"filesystem %s (dev:%s, mount_point:%s, option:%s) "
"is deleted by user(%s)" %
(fs_name, fs_conf['dev_file'],
fs_conf['mount_point'], fs_conf['mount_option'],
user))
def mkfs_on_dev(self, type, dev_file, fs_options=""):
with self.lock:
cls = self._get_fs_type_cls(type)
cls.mkfs(type, dev_file, fs_options)
def test_dict(self):
schema = Schema({
"key1": str, # key1 should be string
"key2": Use(int), # key3 should be in or int in string
"key3": [IntVal(min=10, max=20)],
# key4 is optional,
Optional("key4"): str,
Optional('key5'): Default(IntVal(min=100, max=200), default=100),
DoNotCare(str): object # for all those key we don't care
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key5": 199,
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key5": 199
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key5": 100
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc'
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key4": 'abc',
"key5": 100
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
"key100": 'bbc',
'key200': [123, 23, 334]
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key4": 'abc',
"key5": 100,
"key100": 'bbc',
'key200': [123, 23, 334]
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 123,
"key2": '123',
"key3": [10, 15, 20],
"key4": 223,
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 123,
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
"key100": 'bbc',
'key200': [123, 23, 334]
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 'abc',
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
'key5': 0,
"key100": 'bbc',
'key200': [123, 23, 334]
})
def test_int_value(self):
schema = Schema(IntVal())
self.assertEquals(10, schema.validate(10))
self.assertEquals(10, schema.validate('10'))
with self.assertRaises(SchemaError):
schema.validate('abc')
schema = Schema(IntVal(values=(0, 1)))
self.assertEqual(1, schema.validate(1))
self.assertEqual(1, schema.validate("1"))
with self.assertRaises(SchemaError):
schema.validate(2)
with self.assertRaises(SchemaError):
schema.validate("2")
schema = Schema(IntVal(min=10, max=100))
self.assertEqual(10, schema.validate(10))
self.assertEqual(100, schema.validate('100'))
self.assertEqual(50, schema.validate(50))
with self.assertRaises(SchemaError):
schema.validate(200)
with self.assertRaises(SchemaError):
schema.validate(0)
schema = Schema(IntVal(min=10, max=100, values=(0, 1)))
self.assertEqual(0, schema.validate(0))
self.assertEqual(100, schema.validate(100))
self.assertEqual(50, schema.validate('50'))
with self.assertRaises(SchemaError):
schema.validate(2)
with self.assertRaises(SchemaError):
schema.validate(200)
schema = Schema(IntVal(min=10, values=(0, 1)))
self.assertEqual(200, schema.validate(200))
self.assertEqual(1, schema.validate(1))
with self.assertRaises(SchemaError):
schema.validate(3)
def test_int_value(self):
schema = Schema(IntVal())
self.assertEquals(10, schema.validate(10))
self.assertEquals(10, schema.validate('10'))
with self.assertRaises(SchemaError):
schema.validate('abc')
schema = Schema(IntVal(values=(0, 1)))
self.assertEqual(1, schema.validate(1))
self.assertEqual(1, schema.validate("1"))
with self.assertRaises(SchemaError):
schema.validate(2)
with self.assertRaises(SchemaError):
schema.validate("2")
schema = Schema(IntVal(min=10, max=100))
self.assertEqual(10, schema.validate(10))
self.assertEqual(100, schema.validate('100'))
self.assertEqual(50, schema.validate(50))
with self.assertRaises(SchemaError):
schema.validate(200)
with self.assertRaises(SchemaError):
schema.validate(0)
schema = Schema(IntVal(min=10, max=100, values=(0, 1)))
self.assertEqual(0, schema.validate(0))
self.assertEqual(100, schema.validate(100))
self.assertEqual(50, schema.validate('50'))
with self.assertRaises(SchemaError):
schema.validate(2)
with self.assertRaises(SchemaError):
schema.validate(200)
schema = Schema(IntVal(min=10, values=(0, 1)))
self.assertEqual(200, schema.validate(200))
self.assertEqual(1, schema.validate(1))
with self.assertRaises(SchemaError):
schema.validate(3)
def test_dict(self):
schema = Schema({
"key1":
str, # key1 should be string
"key2":
Use(int), # key3 should be in or int in string
"key3": [IntVal(min=10, max=20)],
# key4 is optional,
Optional("key4"):
str,
Optional('key5'):
Default(IntVal(min=100, max=200), default=100),
DoNotCare(str):
object # for all those key we don't care
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key5": 199,
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key5": 199
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
})
self.assertEqual(data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key5": 100
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc'
})
self.assertEqual(
data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key4": 'abc',
"key5": 100
})
data = schema.validate({
"key1": "abc",
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
"key100": 'bbc',
'key200': [123, 23, 334]
})
self.assertEqual(
data, {
"key1": "abc",
"key2": 123,
"key3": [10, 15, 20],
"key4": 'abc',
"key5": 100,
"key100": 'bbc',
'key200': [123, 23, 334]
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 123,
"key2": '123',
"key3": [10, 15, 20],
"key4": 223,
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 123,
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
"key100": 'bbc',
'key200': [123, 23, 334]
})
with self.assertRaises(SchemaError):
schema.validate({
'key1': 'abc',
"key2": '123',
"key3": [10, 15, 20],
"key4": 'abc',
'key5': 0,
"key100": 'bbc',
'key200': [123, 23, 334]
})
