def test_callback_config_item_in_period(tmp_file, config, another_config):
tmp_file.write(config)
def async_update():
time.sleep(2)
tmp_file.write(another_config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
c = Config(loader, parser)
pc = PeriodicalConfig(c, 3)
item = pc.item("a", caching=1, cb_load=lambda x: int(x) + 1)
pc.load_config(sync=True)
run_in_thread(async_update)
# first time
assert pc.get_value("a") == "1"
assert item.get() == 2
time.sleep(5)
# second time
pc.get_value("a") # trigger update
time.sleep(1)
assert pc.get_value("a") == "2"
assert item.get() == 3
def test_periodical_config_in_period(tmp_file, config, another_config):
tmp_file.write(config)
def async_update():
time.sleep(2)
tmp_file.write(another_config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
c = Config(loader, parser)
pc = PeriodicalConfig(c, 3)
pc.load_config(sync=True)
run_in_thread(async_update)
# first time
assert pc.get_value("a") == "1"
assert pc.get_value("b") == "test"
assert pc.get_int("a") == 1
assert pc.get_string("a") == "1"
assert pc.get_boolean("c")
time.sleep(5)
# second time
pc.get_value("a")
time.sleep(1)
assert pc.get_value("a") == "2"
assert pc.get_value("b") == "test2"
assert pc.get_int("a") == 2
assert pc.get_string("a") == "2"
assert not pc.get_boolean("c")
def test_config_helper_methods(tmp_file, config):
tmp_file.write(config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
c = Config(loader, parser)
c.load_config(sync=True)
assert c.get_boolean("boolean_true_key")
assert not c.get_boolean("boolean_false_key")
assert c.get_int("int_key")
assert c.get_list("list_key") == ["1", "2", "3"]
def test_simple_config(tmp_file, config):
tmp_file.write(config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
c = Config(loader, parser)
c.load_config(sync=True)
assert c.get_value("a") == "1"
assert c.get_value("b") == "test"
assert c.get_int("a") == 1
assert c.get_string("a") == "1"
assert c.get_boolean("c") == True
def test_prefix_config(tmp_file, config):
tmp_file.write(config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
# 增加前缀
c = Config(loader, parser, cb_after_load=lambda x: {'prefix': x})
c.load_config(sync=True)
# first time
assert c.get_value("a") is None
assert c.get_value("prefix.a") == "1"
assert c.get_value("prefix.b") == "test"
assert c.get_int("prefix.a") == 1
assert c.get_string("prefix.a") == "1"
assert c.get_boolean("prefix.c")
prefix_config = PrefixConfig(c, prefix='prefix')
assert prefix_config.get_value("a") == "1"
assert prefix_config.get_value("b") == "test"
assert prefix_config.get_int("a") == 1
assert prefix_config.get_string("a") == "1"
assert prefix_config.get_boolean("c")
def test_config_item_helper_methods(tmp_file, config, another_config):
tmp_file.write(config)
def async_update():
time.sleep(2)
tmp_file.write(another_config)
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
c = Config(loader, parser)
pc = PeriodicalConfig(c, 3)
int_item = pc.int_item("int_key", caching=1)
str_item = pc.str_item("str_key", caching=1)
list_item = pc.list_item("list_key", caching=1)
boolean_true_item = pc.boolean_item("boolean_true_key")
boolean_false_item = pc.boolean_item("boolean_false_key")
pc.load_config(sync=True)
# first batch of assert
assert int_item.get() == 1
assert str_item.get() == "aa"
assert list(list_item.get()) == list(["1", "2", "3"])
assert boolean_true_item.get()
assert not boolean_false_item.get()
run_in_thread(async_update)
time.sleep(5)
pc.get_value("a") # trigger update
time.sleep(1)
assert int_item.get() == 10
assert str_item.get() == "test"
assert list(list_item.get()) == list(["a", "b", "c"])
assert boolean_true_item.get()
assert not boolean_false_item.get()
def init_config():
# import parts
from complexconfig.loader.file_loader import FileLoader
from complexconfig.loader.web_loader import WebLoader
from complexconfig.parser.yaml_parser import YamlParser
from complexconfig.parser.properties_parser import PropertiesParser
from complexconfig.parser.threathunter_json_parser import ThreathunterJsonParser
from complexconfig.config import Config, PeriodicalConfig, CascadingConfig
from complexconfig.configcontainer import configcontainer
# init the global config on /etc/nebula/nebula.conf
global_config_loader = FileLoader("global_config_loader", Global_Conf_FN)
global_config_parser = PropertiesParser("global_config_parser")
# add sniffer prefix
global_config = Config(global_config_loader, global_config_parser, cb_after_load=lambda x: {"sniffer": x})
# init the sniffer module configuration on /etc/nebula/sniffer/sniffer.conf
file_config_loader = FileLoader("file_config_loader", Sniffer_Conf_FN)
file_config_parser = YamlParser("file_config_parser")
# add sniffer prefix
file_config = Config(file_config_loader, file_config_parser, cb_after_load=lambda x: {"sniffer": x})
file_config.load_config(sync=True)
print_with_time("successfully loaded the file config from {}".format(Sniffer_Conf_FN))
web_config_loader = WebLoader("web_config_loader", file_config.get_string("sniffer.web_config.config_url"),
params={"auth": file_config.get_string("sniffer.web_config.auth")})
web_config_parser = ThreathunterJsonParser("web_config_parser")
web_config = Config(web_config_loader, web_config_parser)
web_config.load_config(sync=True)
print "WebLoader: web_config_loader, sniffer.web_config.config_url:{}, params:{}".format(file_config.get_string("sniffer.web_config.config_url"),{"auth": file_config.get_string("sniffer.web_config.auth")})
print_with_time("successfully loaded the web config from {}".format(
file_config.get_string("sniffer.web_config.config_url")))
# build the cascading config
# file config will be updated every half an hour, while the web config will be updated every 5 minute
cascading_config = CascadingConfig(PeriodicalConfig(global_config, 1800), PeriodicalConfig(file_config, 1800),
PeriodicalConfig(web_config, 300))
configcontainer.set_config("sniffer", cascading_config)
print_with_time("successfully loaded config")
Conf_Local_Path = opath.join(Base_Path, "conf", "setting.conf")
Conf_Global_Path = "/etc/nebula/nebula.conf"
Conf_Web_Path = "/etc/nebula/web/settings.conf"
if not os.path.exists(Conf_Global_Path) or not os.path.isfile(
Conf_Global_Path):
print "!!!!Fatal, global conf path {} doesn't exist, using the local path {}".format(
Conf_Global_Path, Conf_Local_Path)
Conf_Global_Path = Conf_Local_Path
if not os.path.exists(Conf_Web_Path) or not os.path.isfile(Conf_Web_Path):
print "!!!!Fatal, web conf path {} doesn't exist, using the local path {}".format(
Conf_Web_Path, Conf_Local_Path)
# init the global config
global_config_loader = FileLoader("global_config_loader", Conf_Global_Path)
global_config_parser = PropertiesParser("global_config_parser")
global_config = Config(global_config_loader, global_config_parser)
global_config.load_config(sync=True)
# init the web config
web_config_loader = FileLoader("web_config_loader", Conf_Web_Path)
web_config_parser = PropertiesParser("web_config_parser")
web_config = Config(web_config_loader, web_config_parser)
web_config.load_config(sync=True)
# build the cascading config
# file config will be updated every half an hour, while the web config
# will be updated every 5 minute
cascading_config = CascadingConfig(global_config, web_config)
configcontainer.set_config("nebula", cascading_config)
_config = configcontainer.get_config("nebula")
def test_cascading_config(tmp_file, config):
# build web config
def login():
result = requests.get("login_url", allow_redirects=False)
assert result.status_code == 200
return {"params": json.loads(result.text)}
loader = WebLoader("web_loader", "config_url", login)
parser = PropertiesParser("parser")
web_config = Config(loader, parser)
web_config.load_config(sync=True)
# build file config
loader = FileLoader("loader", str(tmp_file))
parser = PropertiesParser("parser")
tmp_file.write(config)
file_config = Config(loader, parser)
file_config.load_config(sync=True)
# build cascading config
cascading_config = CascadingConfig(file_config, web_config)
cascading_config.get_int("a")
assert file_config.get_int("a") == 1
assert web_config.get_int("a") == 2
assert cascading_config.get_int("a") == 2
assert file_config.get_string("str_key") == "aa"
assert web_config.get_string("str_key") == "test"
assert cascading_config.get_string("str_key") == "test"
assert file_config.get_value("only_in_config") == "2"
assert web_config.get_value("only_in_config") is None
assert cascading_config.get_value("only_in_config") == "2"