def test_integration(self):
os.environ["BOOL"] = "true"
os.environ["BOOLSTRING"] = "\"false\""
os.environ["DICT"] = "{ \"dict\": \"value\", \"list-in-dict\": [ \"nested-list1\", \"nested-list2\" ] }"
os.environ["FLOAT"] = "1.23"
os.environ["INT"] = "123"
os.environ["KEY"] = "value"
os.environ["LIST"] = "[ \"list1\", \"list2\", { \"dict-in-list\": \"value\" } ]"
os.environ["STRING_WITH_SPECIALS"] = "Test!@#$%^&*()-_=+[]{};:,<.>/?\\\'\"`~"
os.environ["TUPLE"] = "(123, \"string\")"
os.environ["COMPLEX"] = "1+2j"
Pconf.env(
whitelist=[
'KEY',
'INT',
'FLOAT',
'COMPLEX',
'LIST',
'DICT',
'TUPLE',
'BOOL',
'BOOLSTRING',
'STRING_WITH_SPECIALS'
],
parse_values=True,
to_lower=True,
convert_underscores=True
)
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_integration_yaml(self):
self.maxDiff = None
IntegrationBase.result.pop('complex')
IntegrationBase.result.pop('tuple')
Pconf.file('./tests/integration/example.yaml', encoding='yaml')
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_integration_yaml(self):
self.maxDiff = None
IntegrationBase.result.pop("complex")
IntegrationBase.result.pop("tuple")
IntegrationBase.result.pop("secret")
Pconf.file("./tests/integration/example.yaml", encoding="yaml")
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_integration_json(self):
self.maxDiff = None
# Remove values that are not json encodeable
IntegrationBase.result.pop('complex')
IntegrationBase.result.pop('tuple')
Pconf.file('./tests/integration/example.json', encoding='json')
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_integration_json(self):
self.maxDiff = None
# Remove values that are not json encodeable
IntegrationBase.result.pop("complex")
IntegrationBase.result.pop("tuple")
IntegrationBase.result.pop("secret")
Pconf.file("./tests/integration/example.json", encoding="json")
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_argv_get(self, mock_argv):
mocked_argv = MagicMock()
mocked_argv.get.return_value = TEST_ARGV_RESULT
mock_argv.return_value = mocked_argv
arg_name = TEST_ARGV["name"]
Pconf.argv(arg_name)
results = Pconf.get()
for key in TEST_ARGV_RESULT:
self.assertTrue(key in results)
self.assertEqual(results[key], TEST_ARGV_RESULT[key])
def test_file_get(self, mock_file):
mocked_file = MagicMock()
mocked_file.get.return_value = TEST_FILE_RESULT
mock_file.return_value = mocked_file
Pconf.file(path=TEST_FILE_PATH)
results = Pconf.get()
mocked_file.get.assert_called_once()
for key in TEST_FILE_RESULT:
self.assertTrue(key in results)
self.assertEqual(results[key], TEST_FILE_RESULT[key])
def test_env_get(self, mock_env):
mocked_env = MagicMock()
mocked_env.get.return_value = TEST_ENV_RESULT
mock_env.return_value = mocked_env
Pconf.env()
results = Pconf.get()
mocked_env.get.assert_called_once()
for key in TEST_ENV_RESULT:
self.assertTrue(key in results)
self.assertEqual(results[key], TEST_ENV_RESULT[key])
def test_double_backward(self, mock_file, mock_env):
mocked_env = MagicMock()
mocked_env.get.return_value = self.TEST_ENV_RESULT
mock_env.return_value = mocked_env
mocked_file = MagicMock()
mocked_file.get.return_value = self.TEST_FILE_RESULT
mock_file.return_value = mocked_file
Pconf.file(self.TEST_FILE_PATH)
Pconf.env()
Pconf.get()
results = Pconf.get()
expected = {
"file": "result",
"env": "result",
"overlapping": "file",
"deep": {"stillhere": "stillhere", "overlapping": "file"},
}
self.assertEqual(expected, results)
def test_integration(self):
IntegrationBase.result.pop("secret")
sys.argv.append("pconf")
sys.argv.append("--bool")
sys.argv.append("--boolstring")
sys.argv.append("false")
sys.argv.append("--dict")
sys.argv.append(
'{ "dict": "value", "list-in-dict": [ "nested-list1", "nested-list2" ] }'
)
sys.argv.append("--float")
sys.argv.append("1.23")
sys.argv.append("--int")
sys.argv.append("123")
sys.argv.append("--key")
sys.argv.append("value")
sys.argv.append("--list")
sys.argv.append('[ "list1", "list2", { "dict-in-list": "value" } ]')
sys.argv.append("--string-with-specials")
sys.argv.append("Test!@#$%^&*()-_=+[]{};:,<.>/?\\'\"`~")
sys.argv.append("--tuple")
sys.argv.append('(123, "string")')
sys.argv.append("--complex")
sys.argv.append("1+2j")
Pconf.argv(name="--bool", type=bool)
Pconf.argv(name="--boolstring")
Pconf.argv(name="--complex", type=complex)
Pconf.argv(name="--dict", type=dict)
Pconf.argv(name="--float", type=float)
Pconf.argv(name="--int", type=int)
Pconf.argv(name="--key", type=str)
Pconf.argv(name="--list", type=list)
Pconf.argv(name="--string-with-specials")
Pconf.argv(name="--tuple", type=tuple)
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def _init(self, logger):
"""Read the configuration data from files and profile"""
if not self.is_init:
if self.pconf_obj:
self.pconf_obj.clear()
Pconf.clear()
Pconf.env()
dir = self.get_dir()
env_name = self.profile if (len(self.profile)) else os.getenv(
'PY_ENV', 'production')
logger.debug('Configuration dir %s with env %s', dir, env_name)
if path.isdir(dir):
env_file = ''
if env_name == 'production':
env_file = '%s/production_config.json' % dir
elif env_name == 'development':
env_file = '%s/development_conf.json' % dir
elif env_name == 'integration':
env_file = '%s/integration_config.json' % dir
elif env_name == 'test':
env_file = '%s/test_config.json' % dir
elif env_name == 'test_e2e':
env_file = '%s/test_e2e_config.json' % dir
else:
raise ConfigLoadError('Unknown configuration profile: %s' %
env_name)
if not os.path.exists(env_file):
raise ConfigLoadError('Missing configuration file: %s' %
env_file)
Pconf.file(env_file, encoding='json')
base_cfg_file = '%s/config.json' % dir
if not os.path.exists(base_cfg_file):
raise ConfigLoadError('Missing configuration file: %s' %
base_cfg_file)
Pconf.file('%s/config.json' % dir, encoding='json')
else:
raise ConfigLoadError('Missing directory: %s' % dir)
self.is_init = True
self.pconf_obj = Pconf.get()
def test_integration(self):
sys.argv.append("pconf")
sys.argv.append("--bool")
sys.argv.append("--boolstring")
sys.argv.append("false")
sys.argv.append("--dict")
sys.argv.append(
"{ \"dict\": \"value\", \"list-in-dict\": [ \"nested-list1\", \"nested-list2\" ] }"
)
sys.argv.append("--float")
sys.argv.append("1.23")
sys.argv.append("--int")
sys.argv.append("123")
sys.argv.append("--key")
sys.argv.append("value")
sys.argv.append("--list")
sys.argv.append(
"[ \"list1\", \"list2\", { \"dict-in-list\": \"value\" } ]")
sys.argv.append("--string-with-specials")
sys.argv.append("Test!@#$%^&*()-_=+[]{};:,<.>/?\\\'\"\`~")
sys.argv.append("--tuple")
sys.argv.append("(123, \"string\")")
sys.argv.append("--complex")
sys.argv.append("1+2j")
Pconf.argv(name="--bool", type=bool)
Pconf.argv(name="--boolstring")
Pconf.argv(name="--complex", type=complex)
Pconf.argv(name="--dict", type=dict)
Pconf.argv(name="--float", type=float)
Pconf.argv(name="--int", type=int)
Pconf.argv(name="--key", type=str)
Pconf.argv(name="--list", type=list)
Pconf.argv(name="--string-with-specials")
Pconf.argv(name="--tuple", type=tuple)
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_backward(self, mock_file, mock_env):
mocked_env = MagicMock()
mocked_env.get.return_value = self.TEST_ENV_RESULT
mock_env.return_value = mocked_env
mocked_file = MagicMock()
mocked_file.get.return_value = self.TEST_FILE_RESULT
mock_file.return_value = mocked_file
Pconf.file(self.TEST_FILE_PATH)
Pconf.env()
results = Pconf.get()
expected = {
'file': 'result',
'env': 'result',
'overlapping': 'file',
'deep': {
'stillhere': 'stillhere',
'overlapping': 'file'
}
}
self.assertEqual(expected, results)
def test_integration(self):
os.environ["BOOL"] = "true"
os.environ["BOOLSTRING"] = '"false"'
os.environ[
"DICT"
] = '{ "dict": "value", "list-in-dict": [ "nested-list1", "nested-list2" ] }'
os.environ["FLOAT"] = "1.23"
os.environ["INT"] = "123"
os.environ["KEY"] = "value"
os.environ["LIST"] = '[ "list1", "list2", { "dict-in-list": "value" } ]'
os.environ["STRING_WITH_SPECIALS"] = "Test!@#$%^&*()-_=+[]{};:,<.>/?\\'\"`~"
os.environ["TUPLE"] = '(123, "string")'
os.environ["COMPLEX"] = "1+2j"
os.environ["SECRET_FILE"] = "./tests/integration/example_secret"
Pconf.env(
whitelist=[
"KEY",
"INT",
"FLOAT",
"COMPLEX",
"LIST",
"DICT",
"TUPLE",
"BOOL",
"BOOLSTRING",
"STRING_WITH_SPECIALS",
"SECRET_FILE",
],
parse_values=True,
to_lower=True,
convert_underscores=True,
docker_secrets=["SECRET_FILE"],
)
config = Pconf.get()
self.assertEqual(config, IntegrationBase.result)
def test_get_empty_dict_by_default(self):
self.assertEqual(Pconf.get(), {})
def _read_configure(config_filepath, return_empty=False) -> dict:
import os
import pprint
from pconf import Pconf
from ioflow.configure.get_configure_path_from_argv import get_configure_path_from_argv
from ioflow.configure.read_configure import find_best_file_candidate, guess_configure_file_type
# set return_empty to True for not read config from env
# which can prevent unexpected result
# e.g. './configure.json' is not for this app, but for other using
if return_empty:
return {}
config_fileprefix = os.path.splitext(os.path.split(config_filepath)[-1])[0]
default_configure_candidate = [
".".join([config_fileprefix, ext]) for ext in ["yaml", "yml", "json"]
]
builtin_configure_candidate = [
".".join(["./builtin_configure", ext])
for ext in ["yaml", "yml", "json"]
]
default_configure = find_best_file_candidate(default_configure_candidate)
builtin_configure = find_best_file_candidate(builtin_configure_candidate)
active_configure_file = get_configure_path_from_argv()
if not active_configure_file:
active_configure_file = os.getenv("_DEFAULT_CONFIG_FILE",
default_configure)
builtin_configure_file = os.getenv("_BUILTIN_CONFIG_FILE",
builtin_configure)
# Note: this is a safeguard, before using any Pconf function, do execute this
# In case former Pconf usage influence current usage
# which will lead to a hidden and wired bug
Pconf.clear()
# disable read configure from environment
# Pconf.env()
active_configure_file_abs_path = os.path.realpath(active_configure_file)
if not os.path.exists(active_configure_file):
msg = "default configure file is not found! CWD: {}; activate_config: {}; builtin_configure: {}".format(
os.getcwd(), active_configure_file, builtin_configure_file)
print(msg)
raise ValueError(msg)
else:
print(">>> Using configure read from file: {}".format(
active_configure_file_abs_path))
file_encoding = guess_configure_file_type(active_configure_file_abs_path)
Pconf.file(active_configure_file, file_encoding)
# try loading builtin configure file
if builtin_configure_file and os.path.exists(builtin_configure_file):
print(
"loading builtin configure from {}".format(builtin_configure_file))
file_encoding = guess_configure_file_type(builtin_configure_file)
Pconf.file(builtin_configure_file, encoding=file_encoding)
else:
print(">>> builtin configure file is not found!")
# Get all the config values parsed from the sources
config = Pconf.get()
# NOTE: clean Pconf for later brand new use
Pconf.clear()
print("++" * 8, "configure", "++" * 8)
pprint.pprint(config)
return config
def test_clear(self):
Pconf.defaults(TEST_OVERRIDES)
self.assertEqual(Pconf.get(), TEST_OVERRIDES)
Pconf.clear()
self.assertEqual(Pconf.get(), {})
"""Application Config"""
import os
from pconf import Pconf
DIR_PATH = os.path.dirname(os.path.realpath(__file__))
Pconf.defaults({
'bot-move-topic-name': 'bot-move',
"bot-speak-topic-name": 'bot-speak',
"bot-cam-topic-name": 'cam-command'
})
localpath = os.path.join(DIR_PATH, 'dev.json').format()
Pconf.file(localpath, encoding='json')
Pconf.env()
CONFIG = Pconf.get()
MQTT_HOST = CONFIG.get("mqtt-host", '')
MQTT_USER = CONFIG.get("mqtt-user", '')
MQTT_PWD = CONFIG.get("mqtt-pwd", '')
MQTT_PORT = int(CONFIG.get("mqtt-port", -1))
MQTT_MOTOR_MOVE_TOPIC_NAME = CONFIG.get("bot-move-topic-name", '')
MQTT_SPEAK_TOPIC_NAME = CONFIG.get("bot-speak-topic-name", '')
MQTT_CAM_TOPIC_NAME = CONFIG.get("bot-cam-topic-name", '')
WEB_PORT = int(CONFIG.get("PORT", 5000))
def test_overrides_get(self):
Pconf.defaults(TEST_OVERRIDES)
self.assertEqual(Pconf.get(), TEST_OVERRIDES)
def test_defaults_get(self):
Pconf.defaults(TEST_DEFAULTS)
self.assertEqual(Pconf.get(), TEST_DEFAULTS)
text_msg = "今天拉萨的天气。"
predict_result = model.parse([text_msg])
print(predict_result)
if "gunicorn" in sys.modules: # when called by gunicorn in production environment
# disable output log to console
import logging
log = logging.getLogger("werkzeug")
log.disabled = True
Pconf.env(whitelist=["MODEL_PATH"])
config = Pconf.get()
deliverable_server = load_model(config["MODEL_PATH"])
if __name__ == "__main__":
deliverable_server = load_model(sys.argv[1])
warmup_test()
threaded = True
# set threaded to false because keras based models are not thread safety when prediction
# if deliverable_server.model_metadata["model"]["type"] in [
# "keras_h5_model",
# "keras_saved_model",
# ]:
# threaded = False
