class ConfigMergerTest(unittest.TestCase):
def setUp(self):
self.__config_merger = ConfigMerger()
def test_basic(self):
a = {
"hello": {
"world": 1,
"street": "Washingtonova",
},
}
b = {
"hello": {
"zip": "11000",
},
}
result = self.__config_merger.merge(a, b)
self.assertEqual(
{
"hello": {
"world": 1,
"street": "Washingtonova",
"zip": "11000",
}
},
result,
)
def test_overwrite(self):
a = {
"hello": {
"world": 1,
"street": "Washingtonova",
},
}
b = {
"hello": {
"street": "Nova",
"zip": "11000",
},
}
result = self.__config_merger.merge(a, b)
expected_result = {
"hello": {
"world": 1,
"street": "Nova",
"zip": "11000",
}
}
self.assertEqual(expected_result, result)
# strict dict fields order check
self.assertEqual(tuple(expected_result["hello"]),
tuple(result["hello"]))
class ConfigMergerTest(unittest.TestCase):
def setUp(self):
self.__configMerger = ConfigMerger()
def test_basic(self):
a = {
'hello': {
'world': 1,
'street': 'Washingtonova',
},
}
b = {
'hello': {
'zip': '11000',
},
}
result = self.__configMerger.merge(a, b)
self.assertEqual(
{
'hello': {
'world': 1,
'street': 'Washingtonova',
'zip': '11000',
}
}, result)
def test_overwrite(self):
a = {
'hello': {
'world': 1,
'street': 'Washingtonova',
},
}
b = {
'hello': {
'street': 'Nova',
'zip': '11000',
},
}
result = self.__configMerger.merge(a, b)
expectedResult = {
'hello': {
'world': 1,
'street': 'Nova',
'zip': '11000',
}
}
self.assertEqual(expectedResult, result)
# strict dict fields order check
self.assertEqual(tuple(expectedResult['hello']),
tuple(result['hello']))
def modify_raw_config(self, raw_config: dict) -> dict:
project_root_filesystem_path = os.getcwd()
project_root_repo_path = "<not_databricks_repo>"
if is_databricks_repo() and project_root_filesystem_path.startswith(
"/Workspace/Repos"):
project_root_repo_path = project_root_filesystem_path.replace(
"/Workspace/Repos", "/Repos")
project_root_paths = {
"parameters": {
"databricksbundle": {
"project_root": {
"filesystem": {
"path": project_root_filesystem_path,
},
"repo": {
"path": project_root_repo_path,
},
},
}
}
}
return ConfigMerger().merge(raw_config, project_root_paths)
class PyfonyHooks(Hooks):
def __init__(self, bundles: List[Bundle], config_path: str, project_bundles_config_dir: str, app_env: str):
self.__config_merger = ConfigMerger()
self.__bundle_manager = BundleManager(bundles)
self.__config_path = config_path
self.__project_bundles_config_dir = project_bundles_config_dir
self.__app_env = app_env
def start(self, raw_config: dict) -> dict:
bundles_config = self.__bundle_manager.get_bundles_config()
project_bundles_config = self.__bundle_manager.get_project_bundles_config(self.__project_bundles_config_dir)
raw_config = self.__config_merger.merge(self.__config_merger.merge(bundles_config, project_bundles_config), raw_config)
return self.__bundle_manager.modify_raw_config(raw_config)
def services_prepared(
self, services: List[Service], aliases: List[ServiceAlias], parameters: Box
) -> Tuple[List[Service], List[ServiceAlias]]:
return self.__bundle_manager.modify_services(services, aliases, parameters)
def get_custom_parameters(self) -> dict:
pyfony_custom_parameters = {
"project": {
"config_dir": os.path.dirname(self.__config_path),
},
"kernel": {
"environment": self.__app_env,
},
}
return self.__config_merger.merge(super().get_custom_parameters(), pyfony_custom_parameters, False)
def parameters_parsed(self, parameters: Box) -> Box:
return self.__bundle_manager.modify_parameters(parameters)
def container_build_ready(self, container_build: ContainerBuild):
for compiler_pass in self.__bundle_manager.get_compiler_passes(): # type: CompilerPassInterface
compiler_pass.process(container_build)
class ParametersParser:
def __init__(self):
self.__placeholderFiller = PlaceholderFiller()
self.__configMerger = ConfigMerger()
def parse(self, rawParameters: dict, customParameters: dict = None):
if customParameters is not None:
rawParameters = self.__configMerger.merge(rawParameters, customParameters)
parameters = self.__placeholderFiller.fill(rawParameters)
return Box(parameters)
class ConfigLoaderAndMerger:
def __init__(self):
self.__configLoader = ConfigLoader()
self.__configMerger = ConfigMerger()
def loadAndMerge(self, configPaths: List[Path]):
yamlConfig = {}
for configPath in configPaths:
newYamlConfig = self.__configLoader.load(configPath)
if 'imports' in newYamlConfig:
del newYamlConfig['imports']
yamlConfig = self.__configMerger.merge(yamlConfig, newYamlConfig)
return yamlConfig
class ConfigLoaderAndMerger:
def __init__(self):
self.__config_loader = ConfigLoader()
self.__config_merger = ConfigMerger()
def load_and_merge(self, config_paths: List[Path]):
yaml_config = {}
for config_path in config_paths:
new_yaml_config = self.__config_loader.load(config_path)
if "imports" in new_yaml_config:
del new_yaml_config["imports"]
yaml_config = self.__config_merger.merge(yaml_config,
new_yaml_config)
return yaml_config
def __init__(self, bundles: List[Bundle], config_path: str, project_bundles_config_dir: str, app_env: str):
self.__config_merger = ConfigMerger()
self.__bundle_manager = BundleManager(bundles)
self.__config_path = config_path
self.__project_bundles_config_dir = project_bundles_config_dir
self.__app_env = app_env
def __init__(self):
self.__configLoader = ConfigLoader()
self.__configMerger = ConfigMerger()
def __init__(self):
self.__placeholderFiller = PlaceholderFiller()
self.__configMerger = ConfigMerger()
def __init__(self, bundles: List[Bundle]):
self.__bundles = bundles
self.__config_loader = ConfigLoader()
self.__config_merger = ConfigMerger()
class BundleManager:
def __init__(self, bundles: List[Bundle]):
self.__bundles = bundles
self.__config_loader = ConfigLoader()
self.__config_merger = ConfigMerger()
def get_compiler_passes(self) -> List[CompilerPassInterface]:
compiler_passes = []
for bundle in self.__bundles:
compiler_passes += bundle.get_compiler_passes()
return compiler_passes
def get_bundles_config(self) -> dict:
config = dict()
for bundle in self.__bundles:
root_module_name = bundle.__module__[:bundle.__module__.find(".")]
config_base_path = resolve_path(root_module_name) + "/_config"
for config_file_name in bundle.get_config_files():
config_file_path = Path(config_base_path + "/" +
config_file_name)
new_config = self.__config_loader.load(config_file_path)
config = self.__config_merger.merge(config, new_config, False)
return config
def get_project_bundles_config(self, bundles_configs_dir: str) -> dict:
config = dict()
for bundle in self.__bundles:
root_package_name = bundle.__module__[:bundle.__module__.find(".")]
project_bundle_config_path = Path(bundles_configs_dir + "/" +
root_package_name + ".yaml")
if project_bundle_config_path.exists():
project_bundle_config = self.__config_loader.load(
project_bundle_config_path)
config = self.__config_merger.merge(config,
project_bundle_config)
return config
def modify_raw_config(self, raw_config: dict) -> dict:
for bundle in self.__bundles:
raw_config = bundle.modify_raw_config(raw_config)
return raw_config
def modify_services(self, services: List[Service],
aliases: List[ServiceAlias], parameters: Box):
for bundle in self.__bundles:
services, aliases = bundle.modify_services(services, aliases,
parameters)
return services, aliases
def modify_parameters(self, parameters: Box) -> Box:
for bundle in self.__bundles:
parameters = bundle.modify_parameters(parameters)
return parameters
def setUp(self):
self.__configMerger = ConfigMerger()