def test_merge_dictionaries(self):
d1 = {1: 1, 2: 2, 3: 3, 4: {'a': 1, 'b': 'c'}}
d2 = {2: 6, 4: 8, 6: 9, 9: {'c': 12, 'e': 'h'}}
d3 = {4: {'g': 3}, 5: 7, 8: {3: 12, 'k': 'b'}}
with pytest.raises(ValueError):
result = merge_dictionaries([d1, d2, d3])
d2[4] = {8: '8'}
result = merge_dictionaries([d1, d2, d3])
assert result == {1: 1, 2: 6, 3: 3, 4: {8: '8', 'g': 3, 'b': 'c', 'a': 1}, 5: 7, 6: 9, 8: {'k': 'b', 3: 12},
9: {'e': 'h', 'c': 12}}
def test_merge_dictionaries(self):
d1 = {1: 1, 2: 2, 3: 3, 4: {"a": 1, "b": "c"}}
d2 = {2: 6, 4: 8, 6: 9, 9: {"c": 12, "e": "h"}}
d3 = {4: {"g": 3}, 5: 7, 8: {3: 12, "k": "b"}}
with pytest.raises(ValueError):
result = merge_dictionaries([d1, d2, d3])
d2[4] = {8: "8"}
result = merge_dictionaries([d1, d2, d3])
assert result == {
1: 1,
2: 6,
3: 3,
4: {
8: "8",
"g": 3,
"b": "c",
"a": 1
},
5: 7,
6: 9,
8: {
"k": "b",
3: 12
},
9: {
"e": "h",
"c": 12
},
}
d1 = {1: 1, 2: 2, 3: 3, 4: ["a", "b", "c"]}
d2 = {2: 6, 5: 8, 6: 9, 4: ["d", "e"]}
result = merge_dictionaries([d1, d2])
assert result == {1: 1, 2: 6, 3: 3, 4: ["d", "e"], 5: 8, 6: 9}
d1 = {1: 1, 2: 2, 3: 3, 4: ["a", "b", "c"]}
result = merge_dictionaries([d1, d2], merge_lists=True)
assert result == {
1: 1,
2: 6,
3: 3,
4: ["a", "b", "c", "d", "e"],
5: 8,
6: 9,
}
def load_and_merge_json(paths: List[str]) -> dict:
"""Load multiple JSON files and merge into one dictionary
Args:
paths (List[str]): Paths to JSON files
Returns:
dict: Dictionary of merged JSON files
"""
configs = [load_json(path) for path in paths]
return merge_dictionaries(configs)
def load_and_merge_yaml(paths):
# type: (List[str]) -> dict
"""Load multiple YAML files and merge into one dictionary
Args:
paths (List[str]): Paths to YAML files
Returns:
dict: Dictionary of merged YAML files
"""
configs = [load_yaml(path) for path in paths]
return merge_dictionaries(configs)
def load_and_merge_json(paths: List[str], encoding: str = "utf-8") -> Dict:
"""Load multiple JSON files and merge into one dictionary
Args:
paths (List[str]): Paths to JSON files
encoding (str): Encoding of file. Defaults to utf-8.
Returns:
Dict: Dictionary of merged JSON files
"""
configs = [load_json(path, encoding=encoding) for path in paths]
return merge_dictionaries(configs)
def setup_logging(**kwargs):
# type: (Any) -> None
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get('smtp_config_dict', None)
if smtp_config_dict:
smtp_config_found = True
print('Loading smtp configuration customisations from dictionary')
smtp_config_json = kwargs.get('smtp_config_json', '')
if smtp_config_json:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_json)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get('smtp_config_yaml', '')
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError('More than one smtp configuration file given!')
smtp_config_found = True
print('Loading smtp configuration customisations from: %s' %
smtp_config_yaml)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get('logging_config_dict', None)
if logging_config_dict:
logging_config_found = True
print('Loading logging configuration from dictionary')
logging_config_json = kwargs.get('logging_config_json', '')
if logging_config_json:
if logging_config_found:
raise LoggingError(
'More than one logging configuration file given!')
logging_config_found = True
print('Loading logging configuration from: %s' % logging_config_json)
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get('logging_config_yaml', '')
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
'More than one logging configuration file given!')
else:
if not logging_config_yaml:
print('No logging configuration parameter. Using default.')
logging_config_yaml = script_dir_plus_file(
'logging_configuration.yml', setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
'logging_smtp_configuration.yml', setup_logging)
print('Loading base SMTP logging configuration from: %s' %
logging_smtp_config_yaml)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print('Loading logging configuration from: %s' % logging_config_yaml)
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict, logging_smtp_config_dict, smtp_config_dict
])
else:
raise LoggingError(
'SMTP logging configuration file given but not using default logging configuration!'
)
logging.config.dictConfig(logging_config_dict)
def setup_logging(**kwargs: Any) -> None:
"""Setup logging configuration
Args:
**kwargs: See below
logging_config_dict (dict): Logging configuration dictionary OR
logging_config_json (str): Path to JSON Logging configuration OR
logging_config_yaml (str): Path to YAML Logging configuration. Defaults to internal logging_configuration.yml.
smtp_config_dict (dict): Email Logging configuration dictionary if using default logging configuration OR
smtp_config_json (str): Path to JSON Email Logging configuration if using default logging configuration OR
smtp_config_yaml (str): Path to YAML Email Logging configuration if using default logging configuration
Returns:
None
"""
smtp_config_found = False
smtp_config_dict = kwargs.get("smtp_config_dict", None)
if smtp_config_dict:
smtp_config_found = True
print("Loading smtp configuration customisations from dictionary")
smtp_config_json = kwargs.get("smtp_config_json", "")
if smtp_config_json:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_json}"
)
smtp_config_dict = load_json(smtp_config_json)
smtp_config_yaml = kwargs.get("smtp_config_yaml", "")
if smtp_config_yaml:
if smtp_config_found:
raise LoggingError("More than one smtp configuration file given!")
smtp_config_found = True
print(
f"Loading smtp configuration customisations from: {smtp_config_yaml}"
)
smtp_config_dict = load_yaml(smtp_config_yaml)
logging_smtp_config_dict = None
logging_config_found = False
logging_config_dict = kwargs.get("logging_config_dict", None)
if logging_config_dict:
logging_config_found = True
print("Loading logging configuration from dictionary")
logging_config_json = kwargs.get("logging_config_json", "")
if logging_config_json:
if logging_config_found:
raise LoggingError(
"More than one logging configuration file given!")
logging_config_found = True
print(f"Loading logging configuration from: {logging_config_json}")
logging_config_dict = load_json(logging_config_json)
logging_config_yaml = kwargs.get("logging_config_yaml", "")
if logging_config_found:
if logging_config_yaml:
raise LoggingError(
"More than one logging configuration file given!")
else:
if not logging_config_yaml:
print("No logging configuration parameter. Using default.")
logging_config_yaml = script_dir_plus_file(
"logging_configuration.yml", setup_logging)
if smtp_config_found:
logging_smtp_config_yaml = script_dir_plus_file(
"logging_smtp_configuration.yml", setup_logging)
print(
f"Loading base SMTP logging configuration from: {logging_smtp_config_yaml}"
)
logging_smtp_config_dict = load_yaml(logging_smtp_config_yaml)
print(f"Loading logging configuration from: {logging_config_yaml}")
logging_config_dict = load_yaml(logging_config_yaml)
if smtp_config_found:
if logging_smtp_config_dict:
logging_config_dict = merge_dictionaries([
logging_config_dict,
logging_smtp_config_dict,
smtp_config_dict,
])
else:
raise LoggingError(
"SMTP logging configuration file given but not using default logging configuration!"
)
file_only = os.getenv("LOG_FILE_ONLY")
if file_only is not None and file_only.lower() not in [
"false",
"f",
"n",
"no",
"0",
]:
root = logging_config_dict.get("root")
if root is not None:
handlers = root.get("handlers", list())
for i, handler in enumerate(handlers):
if handler.lower() == "console":
del handlers[i]
break
logging.config.dictConfig(logging_config_dict)