def test_get_meta_info(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"
get_repo_file(repo_url, "meta-info.yaml", "meta_info",
augment_meta_info)
repo_info = analyse_repo_url(repo_url)
fname = get_file_location(repo_info, "meta_info")
with open(fname, "r") as file:
output = file.read()
file_name = "meta_info.expected"
file_url = "https://raw.githubusercontent.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream/master/meta-info.yaml"
meta_info_output = urlopen(file_url).read()
with open(file_name, "w") as file:
file.write(meta_info_output)
augment_meta_info(file_name)
with open(file_name, "r") as file:
expected_output = file.read()
self.assertEqual(output, expected_output)
def test_analyse_repo_url(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/simple_grid_yaml_compiler"
expected_output = {
"org_name": "WLCG-Lightweight-Sites",
"repo_name": "simple_grid_yaml_compiler",
"branch_name": "master"
}
self.assertDictEqual(analyse_repo_url(repo_url), expected_output)
def phase_5(phase_4_output, runtime_vars, yaml):
phase_4_output_file = open(phase_4_output.name, 'r')
phase_5_output_file = open("./.temp/phase_5_output.yaml", 'w')
data = yaml.load(phase_4_output_file)
phase_5_output = {}
for data_section in data:
if data_section == 'lightweight_components':
updated_components = []
for lightweight_component in data[data_section]:
print "Component: " + lightweight_component['name']
updated_component = {}
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_processor.get_repository_file(
repo_url, 'config-schema.yaml')
repo_processor.get_repository_file(
repo_url, 'meta-info.yaml', True)
repo_info = repo_processor.analyse_repo_url(repo_url)
config_schema_file_name = repo_processor.generate_file_name(
repo_info, 'config-schema.yaml')
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file = repo_processor.generate_file_name(
repo_info, 'meta-info.yaml')
meta_info_parent_name = repo_processor.generate_meta_info_parent_name(
meta_info_file)
meta_info = data[meta_info_parent_name]
default_data_file_name = repo_processor.generate_file_name(
repo_info, 'default-data.yaml')
default_data_runtime_file = open(
default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
for l in default_data_file.readlines():
default_data_runtime_file.write(l)
default_data_file.close()
default_data_runtime_file.close()
config_schema_file.close()
augmented_config = processor_config_schemas.process_config_schema(
lightweight_component[component_section],
config_schema_file, default_data_runtime_file,
meta_info)
updated_component[component_section] = augmented_config
else:
updated_component[
component_section] = lightweight_component[
component_section]
updated_components.append(updated_component)
phase_5_output[data_section] = updated_components
else:
phase_5_output[data_section] = data[data_section]
yaml.dump(phase_5_output, phase_5_output_file)
phase_5_output_file.close()
return phase_5_output_file
def phase_7(yamale_flie, config_file):
yaml = YAML()
base_repo_info = repo_processor.analyse_repo_url(constants.BASE_REPO_URL)
config_schema_file = repo_processor.get_file_location(
base_repo_info, "config_schema")
lc_schemas = set()
config_file = open(config_file, "r")
config = yaml.load(config_file)
for lc in config["lightweight_components"]:
lc_url = lc["repository_url"]
lc_info = repo_processor.analyse_repo_url(lc_url)
lc_schema = repo_processor.get_file_location(lc_info, "config_schema")
lc_schemas.add(lc_schema)
yamale_converter.yamale_converter(config_schema_file, yamale_flie,
lc_schemas)
def phase_5(phase_4_output, runtime_vars):
phase_4_output_file = open(phase_4_output.name, 'r')
data = yaml.load(phase_4_output_file)
output = {}
for data_section in data:
if data_section == 'lightweight_components':
updated_components = []
for lightweight_component in data[data_section]:
print "Component: " + lightweight_component['name']
updated_component = {}
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_processor.get_config_schema(repo_url)
repo_processor.get_meta_info(repo_url)
repo_info = repo_processor.analyse_repo_url(repo_url)
config_schema_file_name = repo_processor.generate_config_schema_file_name(
repo_info)
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file_name = repo_processor.generate_meta_info_file_name(
repo_info)
meta_info_file = open(meta_info_file_name, 'r')
default_data_file_name = repo_processor.generate_default_file_name(
repo_info)
default_data_runtime_file = open(
default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
for l in default_data_file.readlines():
default_data_runtime_file.write(l)
default_data_file.close()
default_data_runtime_file.close()
config_schema_file.close()
meta_info_file.close()
augmented_config = processor_config_schemas.process_config_schema(
lightweight_component[component_section],
config_schema_file, default_data_runtime_file,
meta_info_file)
updated_component[component_section] = augmented_config
else:
updated_component[
component_section] = lightweight_component[
component_section]
updated_components.append(updated_component)
output[data_section] = updated_components
else:
output[data_section] = data[data_section]
print output
return output
def test_get_config_schema(self):
repo_url = "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream"
get_repo_file(repo_url, "config-schema.yaml", "config_schema")
repo_info = analyse_repo_url(repo_url)
fname = get_file_location(repo_info, "config_schema")
with open(fname, "r") as file:
output = file.read()
file_url = "https://raw.githubusercontent.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream/master/config-schema.yaml"
expected_output = urlopen(file_url).read()
self.assertEqual(output, expected_output)
def phase_5(phase_4_output, runtime_vars, yaml):
phase_4_output_file = open(phase_4_output.name, 'r')
phase_5_output_file = open("./.temp/phase_5_output.yaml", 'w')
data = yaml.load(phase_4_output_file)
base_repo_info = repo_processor.analyse_repo_url(
constants.BASE_REPO_URL, constants.BASE_REPO_REVISION)
base_repo_file = lambda x: repo_processor.get_file_location(
base_repo_info, x)
data = processor_config_schemas.process_complete_schema(
config=data,
config_schema_file=base_repo_file("config_schema"),
defaults_file=base_repo_file("defaults"),
meta_info_file=base_repo_file("meta_info"))
phase_5_output = {}
for data_section in data:
if data_section == 'lightweight_components':
updated_components = []
for lightweight_component in data[data_section]:
print "Component: " + lightweight_component['name']
updated_component = {}
for component_section in lightweight_component:
if component_section == 'config':
repo_url = lightweight_component['repository_url']
repo_version = lightweight_component[
'repository_revision']
repo_processor.get_repo_file(repo_url,
"config-schema.yaml",
"config_schema",
branch=repo_version)
repo_processor.get_repo_file(
repo_url,
"meta-info.yaml",
"meta_info",
branch=repo_version,
post_func=repo_processor.augment_meta_info)
repo_info = repo_processor.analyse_repo_url(
repo_url, repo_version)
config_schema_file_name = repo_processor.get_file_location(
repo_info, "config_schema")
config_schema_file = open(config_schema_file_name, 'r')
meta_info_file = repo_processor.get_file_location(
repo_info, "meta_info")
meta_info_parent_name = repo_processor.generate_meta_info_parent_name(
meta_info_file)
meta_info = data[meta_info_parent_name]
default_data_file_name = repo_processor.get_file_location(
repo_info, "defaults")
default_data_runtime_file = open(
default_data_file_name + ".runtime", 'w')
default_data_file = open(default_data_file_name, 'r')
default_data_runtime_file.write(runtime_vars)
for l in default_data_file.readlines():
default_data_runtime_file.write(l)
default_data_file.close()
default_data_runtime_file.close()
config_schema_file.close()
augmented_config = processor_config_schemas.process_config_schema(
lightweight_component[component_section],
config_schema_file, default_data_runtime_file,
meta_info)
updated_component[component_section] = augmented_config
else:
updated_component[
component_section] = lightweight_component[
component_section]
updated_components.append(updated_component)
phase_5_output[data_section] = updated_components
else:
phase_5_output[data_section] = data[data_section]
yaml.dump(phase_5_output, phase_5_output_file)
phase_5_output_file.close()
return phase_5_output_file