def phase_1(site_level_configuration_file):
# fetch repo and get default_values.yaml
repo_processor.get_base_files(constants.BASE_REPO_URL,
constants.BASE_REPO_REVISION)
main_default_values_file = repo_processor.get_repo_file(
constants.BASE_REPO_URL,
"site_level_configuration_defaults.yaml",
"site_level_defaults",
branch=constants.BASE_REPO_REVISION)
repo_urls = lexemes.get_repo_list(site_level_configuration_file)
print(repo_urls)
file_names_repository_default = [main_default_values_file]
file_names_repository_meta = []
for url in repo_urls:
file_names_repository_default.append(
repo_processor.get_repo_file(url['url'],
'default-data.yaml',
"defaults",
branch=url['revision']))
file_names_repository_meta.append(
repo_processor.get_repo_file(
url['url'],
"meta-info.yaml",
"meta_info",
branch=url['revision'],
post_func=repo_processor.augment_meta_info))
all_includes = file_names_repository_meta + file_names_repository_default
includes_yaml_file = yaml_augmentation.add_include_statements(
all_includes, site_level_configuration_file)
return includes_yaml_file, repo_urls
def test_get_meta_info(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"
get_repo_file(repo_url, "meta-info.yaml", "meta_info",
augment_meta_info)
repo_info = analyse_repo_url(repo_url)
fname = get_file_location(repo_info, "meta_info")
with open(fname, "r") as file:
output = file.read()
file_name = "meta_info.expected"
file_url = "https://raw.githubusercontent.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream/master/meta-info.yaml"
meta_info_output = urlopen(file_url).read()
with open(file_name, "w") as file:
file.write(meta_info_output)
augment_meta_info(file_name)
with open(file_name, "r") as file:
expected_output = file.read()
self.assertEqual(output, expected_output)
def test_get_config_schema(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"
get_repo_file(repo_url, "config-schema.yaml", "config_schema")
repo_info = analyse_repo_url(repo_url)
fname = get_file_location(repo_info, "config_schema")
with open(fname, "r") as file:
output = file.read()
file_url = "https://raw.githubusercontent.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream/master/config-schema.yaml"
expected_output = urlopen(file_url).read()
self.assertEqual(output, expected_output)
def phase_1(site_level_configuration_file):
# fetch repo and get default_values.yaml
repo_processor.get_base_files(constants.BASE_REPO_URL)
main_default_values_file = repo_processor.get_repo_file(
"https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults",
"site_level_configuration_defaults.yaml", "defaults")
repo_urls = lexemes.get_repo_list(site_level_configuration_file)
print(repo_urls)
file_names_repository_default = [main_default_values_file]
file_names_repository_meta = []
for url in repo_urls:
file_names_repository_default.append(
repo_processor.get_repo_file(url, 'default-data.yaml', "defaults"))
file_names_repository_meta.append(
repo_processor.get_repo_file(url, "meta-info.yaml", "meta_info",
repo_processor.augment_meta_info))
all_includes = file_names_repository_meta + file_names_repository_default
includes_yaml_file = yaml_augmentation.add_include_statements(
all_includes, site_level_configuration_file)
return includes_yaml_file, repo_urls
def test_get_default_values(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/simple_grid_site_defaults"
defaults_file = "site_level_configuration_defaults.yaml"
fname = get_repo_file(repo_url, defaults_file, "defaults")
with open(fname, "r") as file:
output = file.read()
file_url = "https://raw.githubusercontent.com/WLCG-Lightweight-Sites/simple_grid_site_defaults/master/site_level_configuration_defaults.yaml"
expected_output = urlopen(file_url).read()
self.assertEqual(output, expected_output)
def phase_5(phase_4_output, runtime_vars, yaml):
phase_4_output_file = open(phase_4_output.name, 'r')
phase_5_output_file = open("./.temp/phase_5_output.yaml", 'w')
data = yaml.load(phase_4_output_file)
base_repo_info = repo_processor.analyse_repo_url(
constants.BASE_REPO_URL, constants.BASE_REPO_REVISION)
base_repo_file = lambda x: repo_processor.get_file_location(
base_repo_info, x)
data = processor_config_schemas.process_complete_schema(
config=data,
config_schema_file=base_repo_file("config_schema"),
defaults_file=base_repo_file("defaults"),
meta_info_file=base_repo_file("meta_info"))
phase_5_output = {}
for data_section in data:
if data_section == 'lightweight_components':
updated_components = []
for lightweight_component in data[data_section]:
print "Component: " + lightweight_component['name']
updated_component = {}
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_version = lightweight_component[
'repository_revision']
repo_processor.get_repo_file(repo_url,
"config-schema.yaml",
"config_schema",
branch=repo_version)
repo_processor.get_repo_file(
repo_url,
"meta-info.yaml",
"meta_info",
branch=repo_version,
post_func=repo_processor.augment_meta_info)
repo_info = repo_processor.analyse_repo_url(
repo_url, repo_version)
config_schema_file_name = repo_processor.get_file_location(
repo_info, "config_schema")
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file = repo_processor.get_file_location(
repo_info, "meta_info")
meta_info_parent_name = repo_processor.generate_meta_info_parent_name(
meta_info_file)
meta_info = data[meta_info_parent_name]
default_data_file_name = repo_processor.get_file_location(
repo_info, "defaults")
default_data_runtime_file = open(
default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
for l in default_data_file.readlines():
default_data_runtime_file.write(l)
default_data_file.close()
default_data_runtime_file.close()
config_schema_file.close()
augmented_config = processor_config_schemas.process_config_schema(
lightweight_component[component_section],
config_schema_file, default_data_runtime_file,
meta_info)
updated_component[component_section] = augmented_config
else:
updated_component[
component_section] = lightweight_component[
component_section]
updated_components.append(updated_component)
phase_5_output[data_section] = updated_components
else:
phase_5_output[data_section] = data[data_section]
yaml.dump(phase_5_output, phase_5_output_file)
phase_5_output_file.close()
return phase_5_output_file