def do_list_imp(self, what=None, stream=sys.stdout):
if what is None:
aYaml.writeAsYaml(self, stream)
list_to_do = list()
if isinstance(what, str):
list_to_do.append(what)
elif isinstance(what, list):
list_to_do.extend(what)
whole_sections_to_write = list()
individual_items_to_write = list()
for item_to_do in list_to_do:
if utils.guid_re.match(item_to_do):
translated_iids, orphaned_guids = self.items_table.iids_from_guids([item_to_do])
whole_sections_to_write.append({item_to_do: translated_iids})
elif item_to_do == "define":
whole_sections_to_write.append(aYaml.YamlDumpDocWrap(config_vars, '!define', "Definitions", explicit_start=True, sort_mappings=True))
elif item_to_do == "index":
whole_sections_to_write.append(aYaml.YamlDumpDocWrap(self.items_table.repr_for_yaml(), '!index', "Installation index", explicit_start=True, sort_mappings=True))
elif item_to_do == "guid":
guid_dict = dict()
all_guids = self.items_table.get_detail_values_by_name_for_all_iids("guid")
for a_guid in all_guids:
translated_iids, orphaned_guids = self.items_table.iids_from_guids([a_guid])
guid_dict[a_guid] = translated_iids
whole_sections_to_write.append(aYaml.YamlDumpDocWrap(guid_dict, '!guid', "guid to IID", explicit_start=True, sort_mappings=True))
else:
individual_items_to_write.append(item_to_do)
aYaml.writeAsYaml(whole_sections_to_write + self.repr_for_yaml(individual_items_to_write), stream)
def __call__(self, *args, **kwargs) -> None:
short_index_data = self.items_table.get_data_for_short_index(
) # iid, name, version_mac, version_win, install_guid, remove_guid
short_index_dict = defaultdict(dict)
builtin_iids = list(config_vars["SPECIAL_BUILD_IN_IIDS"])
for data_line in short_index_data:
data_dict = dict(data_line)
IID = data_dict['iid']
if IID not in builtin_iids:
if data_dict['name']:
short_index_dict[IID]['name'] = data_dict['name']
if data_dict['version_mac'] == data_dict['version_win']:
short_index_dict[IID]['version'] = data_dict['version_mac']
else:
if data_dict['version_mac']:
short_index_dict[IID]['Mac'] = {
'version': data_dict['version_mac']
}
if data_dict['version_win']:
short_index_dict[IID]['Win'] = {
'version': data_dict['version_win']
}
if data_dict['install_guid']:
if data_dict['remove_guid'] != data_dict[
'install_guid']: # found uninstall gui
short_index_dict[IID]['guid'] = list(
(data_dict['install_guid'],
data_dict['remove_guid']))
else:
short_index_dict[IID]['guid'] = data_dict[
'install_guid']
defines_dict = config_vars.repr_for_yaml(which_vars=list(
config_vars['SHORT_INDEX_FILE_VARS']),
resolve=True,
ignore_unknown_vars=False)
defines_yaml_doc = aYaml.YamlDumpDocWrap(defines_dict,
'!define',
"Definitions",
explicit_start=True,
sort_mappings=True)
index_yaml_doc = aYaml.YamlDumpDocWrap(value=short_index_dict,
tag="!index",
explicit_start=True,
explicit_end=False,
sort_mappings=True,
include_comments=False)
with utils.utf8_open_for_write(self.short_index_yaml_path, "w") as wfd:
aYaml.writeAsYaml(defines_yaml_doc, wfd)
aYaml.writeAsYaml(index_yaml_doc, wfd)
def repr_for_yaml(self, what=None):
""" Create representation of self suitable for printing as yaml.
parameter 'what' is a list of identifiers to represent. If 'what'
is None (the default) create representation of everything.
InstlInstanceBase object is represented as two yaml documents:
one for define (tagged !define), one for the index (tagged !index).
"""
retVal = list()
all_iids = self.items_table.get_all_iids()
all_vars = sorted(config_vars.keys())
if what is None: # None is all
what = all_vars + all_iids
defines = OrderedDict()
indexes = OrderedDict()
unknowns = list()
for identifier in what:
if identifier in all_vars:
defines.update(
{identifier: config_vars.repr_var_for_yaml(identifier)})
elif identifier in all_iids:
indexes.update({
identifier:
self.items_table.repr_item_for_yaml(identifier)
})
else:
unknowns.append(
aYaml.YamlDumpWrap(value="UNKNOWN VARIABLE",
comment=identifier +
" is not in variable list"))
if defines:
retVal.append(
aYaml.YamlDumpDocWrap(defines,
'!define',
"Definitions",
explicit_start=True,
sort_mappings=True))
if indexes:
retVal.append(
aYaml.YamlDumpDocWrap(indexes,
'!index',
"Installation index",
explicit_start=True,
sort_mappings=True))
if unknowns:
retVal.append(
aYaml.YamlDumpDocWrap(unknowns,
'!unknowns',
"Installation index",
explicit_start=True,
sort_mappings=True))
return retVal
def write_require_file(self, file_path, require_dict):
with utils.utf8_open_for_write(file_path, "w") as wfd:
define_dict = aYaml.YamlDumpDocWrap(
{"REQUIRE_REPO_REV": config_vars["MAX_REPO_REV"].str()},
'!define',
"definitions",
explicit_start=True,
sort_mappings=True)
require_dict = aYaml.YamlDumpDocWrap(require_dict,
'!require',
"requirements",
explicit_start=True,
sort_mappings=True)
aYaml.writeAsYaml((define_dict, require_dict), wfd)
def test_write(self):
as_yaml = self.it.repr_for_yaml()
as_yaml_doc = aYaml.YamlDumpDocWrap(as_yaml, '!index')
as_yaml_doc.ReduceOneItemLists()
with open(self.out_file_path, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
aYaml.writeAsYaml(as_yaml_doc, wfd)
def do_read_yaml(self):
config_vars["OUTPUT_FORMAT"] = "yaml"
config_vars_yaml_obj = config_vars.repr_for_yaml()
config_vars_yaml = aYaml.YamlDumpDocWrap(config_vars_yaml_obj,
'!define',
"Definitions",
explicit_start=True,
sort_mappings=True,
include_comments=False)
self.output_data.append(config_vars_yaml)
index_yaml_obj = self.items_table.repr_for_yaml()
index_yaml = aYaml.YamlDumpDocWrap(index_yaml_obj,
'!index',
"Installation index",
explicit_start=True,
sort_mappings=True,
include_comments=False)
self.output_data.append(index_yaml)
def write_history(self):
selected_tab = self.notebook.tab(self.notebook.select(), option='text')
config_vars["SELECTED_TAB"] = selected_tab
which_vars_for_yaml = config_vars.get("__GUI_CONFIG_FILE_VARS__", []).list()
the_list_yaml_ready= config_vars.repr_for_yaml(which_vars=which_vars_for_yaml, resolve=False, ignore_unknown_vars=True)
the_doc_yaml_ready = aYaml.YamlDumpDocWrap(the_list_yaml_ready, '!define', "Definitions", explicit_start=True, sort_mappings=True)
with utils.utf8_open_for_write(config_vars["INSTL_GUI_CONFIG_FILE_NAME"].str(), "w") as wfd:
aYaml.writeAsYaml(the_doc_yaml_ready, wfd)
def __call__(self, *args, **kwargs) -> None:
if "REPO_REV_FILE_VARS" not in config_vars:
# must have a list of variable names to write to the repo-rev file
raise ValueError("REPO_REV_FILE_VARS must be defined")
repo_rev_vars = list(config_vars["REPO_REV_FILE_VARS"]) # list of configVars to write to the repo-rev file
# check that the variable names from REPO_REV_FILE_VARS do not contain
# names that must not be made public
dangerous_intersection = set(repo_rev_vars).intersection(
{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "PRIVATE_KEY", "PRIVATE_KEY_FILE"})
if dangerous_intersection:
log.warning("found", str(dangerous_intersection), "in REPO_REV_FILE_VARS, aborting")
raise ValueError(f"file REPO_REV_FILE_VARS {dangerous_intersection} and so is forbidden to upload")
use_zlib = bool(config_vars.get("USE_ZLIB", "False")) # should we consider zipped files or not
zip_extension = ""
if use_zlib:
zip_extension = config_vars.get("WZLIB_EXTENSION", ".wzip").str()
revision_instl_folder_path = Path(config_vars["UPLOAD_REVISION_INSTL_FOLDER"])
# create checksum for the main info_map file, either wzipped or not
main_info_map_file_name = "info_map.txt"+zip_extension
main_info_map_file = revision_instl_folder_path.joinpath(main_info_map_file_name)
main_info_map_checksum = utils.get_file_checksum(main_info_map_file)
config_vars["INFO_MAP_FILE_URL"] = "$(BASE_LINKS_URL)/$(REPO_NAME)/$(__CURR_REPO_FOLDER_HIERARCHY__)/instl/"+main_info_map_file_name
config_vars["INFO_MAP_CHECKSUM"] = main_info_map_checksum
# create checksum for the main index.yaml file, either wzipped or not
index_file_name = "index.yaml"+zip_extension
index_file_path = revision_instl_folder_path.joinpath(index_file_name)
config_vars["INDEX_CHECKSUM"] = utils.get_file_checksum(index_file_path)
config_vars["INDEX_URL"] = "$(BASE_LINKS_URL)/$(REPO_NAME)/$(__CURR_REPO_FOLDER_HIERARCHY__)/instl/"+index_file_name
short_index_file_name = "short-index.yaml"
short_index_file_path = revision_instl_folder_path.joinpath(short_index_file_name)
config_vars["SHORT_INDEX_CHECKSUM"] = utils.get_file_checksum(short_index_file_path)
config_vars["SHORT_INDEX_URL"] = "$(BASE_LINKS_URL)/$(REPO_NAME)/$(__CURR_REPO_FOLDER_HIERARCHY__)/instl/"+short_index_file_name
config_vars["INSTL_FOLDER_BASE_URL"] = "$(BASE_LINKS_URL)/$(REPO_NAME)/$(__CURR_REPO_FOLDER_HIERARCHY__)/instl"
config_vars["REPO_REV_FOLDER_HIERARCHY"] = "$(__CURR_REPO_FOLDER_HIERARCHY__)"
# check that all variables are present
# <class 'list'>: ['INSTL_FOLDER_BASE_URL', 'REPO_REV_FOLDER_HIERARCHY', 'SYNC_BASE_URL']
missing_vars = [var for var in repo_rev_vars if var not in config_vars]
if missing_vars:
raise ValueError(f"{missing_vars} are missing cannot write repo rev file")
# create yaml out of the variables
variables_as_yaml = config_vars.repr_for_yaml(repo_rev_vars)
repo_rev_yaml_doc = aYaml.YamlDumpDocWrap(variables_as_yaml, '!define', "",
explicit_start=True, sort_mappings=True)
repo_rev_file_path = config_vars["UPLOAD_REVISION_REPO_REV_FILE"]
with utils.utf8_open_for_write(repo_rev_file_path, "w") as wfd:
aYaml.writeAsYaml(repo_rev_yaml_doc, out_stream=wfd, indentor=None, sort=True)
log.info(f"""create {repo_rev_file_path}""")
def do_short_index(self):
short_index_data = self.items_table.get_data_for_short_index(
) # IID, GUID, NAME, VERSION, generation
short_index_dict = defaultdict(dict)
for data_line in short_index_data:
short_index_dict[data_line[0]]['guid'] = data_line[1]
if data_line[4] and data_line[1] != data_line[4]: # uninstall gui
short_index_dict[data_line[0]]['guid'] = list(
(data_line[1], data_line[4]))
if data_line[2]:
short_index_dict[data_line[0]]['name'] = data_line[2]
if data_line[3] or data_line[4]:
short_index_dict[data_line[0]]['version'] = data_line[3]
defines_dict = config_vars.repr_for_yaml(which_vars=['AUXILIARY_IIDS'],
resolve=True,
ignore_unknown_vars=False)
defines_yaml_doc = aYaml.YamlDumpDocWrap(defines_dict,
'!define',
"Definitions",
explicit_start=True,
sort_mappings=True)
index_yaml_doc = aYaml.YamlDumpDocWrap(value=short_index_dict,
tag="!index",
explicit_start=True,
explicit_end=False,
sort_mappings=True,
include_comments=False)
def __command_output(self, _as_yaml_doc):
out_file_path = config_vars.get("__MAIN_OUT_FILE__", None).Path()
with utils.write_to_file_or_stdout(out_file_path) as wfd:
aYaml.writeAsYaml(defines_yaml_doc, wfd)
aYaml.writeAsYaml(index_yaml_doc, wfd)
from functools import partial
self.command_output = partial(__command_output, self, index_yaml_doc)
def test_readFile(self):
input_file_path = Path(__file__).parent.joinpath("test_input.yaml")
out_file_path = Path(__file__).parent.joinpath("test_out.yaml")
expected_file_path = Path(__file__).parent.joinpath(
"expected_output.yaml")
reader = ConfigVarYamlReader()
reader.read_yaml_file(input_file_path)
variables_as_yaml = config_vars.repr_for_yaml()
yaml_doc = aYaml.YamlDumpDocWrap(variables_as_yaml,
'!define',
"",
explicit_start=True,
sort_mappings=True)
with open(out_file_path, "w") as wfd:
aYaml.writeAsYaml(yaml_doc, wfd)
out_lines = normalize_yaml_lines(out_file_path)
expected_lines = normalize_yaml_lines(expected_file_path)
self.assertEqual(out_lines, expected_lines)