def read_admin_config_file(self):
config_path = var_stack.ResolveVarToStr("ADMIN_GUI_CONFIG_FILE", default="")
if os.path.isfile(config_path):
var_stack.get_configVar_obj("__SEARCH_PATHS__").clear_values() # so __include__ file will not be found on old paths
self.read_yaml_file(config_path)
self.admin_config_file_dirty = False
else:
print("File not found:", config_path)
def create_download_instructions(self):
self.instlObj.batch_accum.set_current_section('sync')
self.instlObj.batch_accum += self.instlObj.platform_helper.progress("Starting sync from $(SYNC_BASE_URL)")
self.sync_base_url = var_stack.ResolveVarToStr("SYNC_BASE_URL")
self.instlObj.batch_accum += self.instlObj.platform_helper.new_line()
for iid in var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"):
with self.install_definitions_index[iid].push_var_stack_scope():
for source_var in var_stack.get_configVar_obj("iid_source_var_list"):
source = var_stack.ResolveVarToList(source_var)
self.p4_sync_for_source(source)
def mark_required_items(self):
""" Mark all files that are needed for installation.
Folders containing these these files are also marked.
All required items are written to required_info_map.txt for reference.
"""
for iid in var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"):
with self.instlObj.install_definitions_index[iid].push_var_stack_scope():
for source_var in var_stack.get_configVar_obj("iid_source_var_list"):
source = var_stack.ResolveVarToList(source_var)
self.instlObj.info_map_table.mark_required_for_source(source)
self.instlObj.info_map_table.mark_required_completion()
required_file_path = var_stack.ResolveVarToStr("REQUIRED_INFO_MAP_PATH")
required_items_list = self.instlObj.info_map_table.get_required_items()
self.instlObj.info_map_table.write_to_file(in_file=required_file_path, items_list=required_items_list)
def sort_install_items_by_target_folder(self, instlObj):
for IID in self.full_install_items:
with instlObj.install_definitions_index[IID].push_var_stack_scope():
folder_list_for_idd = [folder for folder in var_stack["iid_folder_list"]]
if folder_list_for_idd:
for folder in folder_list_for_idd:
norm_folder = os.path.normpath(folder)
self.install_items_by_target_folder[norm_folder].append(IID)
else: # items that need no copy
for source_var in var_stack.get_configVar_obj("iid_source_var_list"):
source = var_stack.ResolveVarToList(source_var)
relative_sync_folder = instlObj.relative_sync_folder_for_source(source)
sync_folder = os.path.join("$(LOCAL_REPO_SYNC_DIR)", relative_sync_folder)
self.no_copy_items_by_sync_folder[sync_folder].append(IID)
def create_sync_instructions(self):
super().create_sync_instructions()
self.instlObj.batch_accum += self.instlObj.platform_helper.progress("Starting sync from $(SYNC_BASE_URL)")
self.instlObj.batch_accum += self.instlObj.platform_helper.mkdir("$(LOCAL_SYNC_DIR)")
self.instlObj.batch_accum += self.instlObj.platform_helper.cd("$(LOCAL_SYNC_DIR)")
self.instlObj.batch_accum += " ".join(('"$(SVN_CLIENT_PATH)"', "co", '"$(BOOKKEEPING_DIR_URL)"', '"$(REL_BOOKKEEPING_PATH)"', "--revision", "$(REPO_REV)", "--depth", "infinity"))
self.instlObj.batch_accum += self.instlObj.platform_helper.progress("instl folder file $(BOOKKEEPING_DIR_URL)?p=$(REPO_REV)")
for iid in var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"):
with self.install_definitions_index[iid].push_var_stack_scope():
for source_var in var_stack.get_configVar_obj("iid_source_var_list"):
source = var_stack.ResolveVarToList(source_var)
self.instlObj.batch_accum += self.create_svn_sync_instructions_for_source(source)
self.instlObj.batch_accum += self.instlObj.platform_helper.progress("Sync {}".format(var_stack.ResolveVarToStr("iid_name")))
for iid in var_stack.ResolveVarToList("__ORPHAN_INSTALL_TARGETS__"):
self.instlObj.batch_accum += self.instlObj.platform_helper.echo("Don't know how to sync " + iid)
self.instlObj.batch_accum += self.instlObj.platform_helper.echo("from $(SYNC_BASE_URL)")
def defaults_help(self):
defaults_folder_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults")
for yaml_file in os.listdir(defaults_folder_path):
if fnmatch.fnmatch(yaml_file, '*.yaml') and yaml_file != "P4.yaml": # hack to not read the P4 defaults
self.instlObj.read_yaml_file(os.path.join(defaults_folder_path, yaml_file))
defaults_list = [("Variable name", "Raw value", "Resolved value"),
("_____________", "_________", "______________")]
for var in sorted(var_stack):
if not var.startswith("__"):
cv = var_stack.get_configVar_obj(var)
raw_value = " ".join([value for value in cv])
resolved_value = var_stack.ResolveVarToStr(var)
if raw_value != resolved_value:
defaults_list.append((var, raw_value, resolved_value))
else:
defaults_list.append((var, raw_value))
col_format = utils.gen_col_format(utils.max_widths(defaults_list))
for res_line in defaults_list:
print(col_format[len(res_line)].format(*res_line))
def create_remove_instructions(self):
have_info_path = var_stack.ResolveVarToStr("HAVE_INFO_MAP_PATH")
if not os.path.isfile(have_info_path):
have_info_path = var_stack.ResolveVarToStr("SITE_HAVE_INFO_MAP_PATH")
self.read_info_map_from_file(have_info_path)
self.batch_accum.set_current_section("remove")
self.batch_accum += self.platform_helper.progress("Starting remove")
sorted_target_folder_list = sorted(
self.all_items_by_target_folder, key=lambda fold: var_stack.ResolveStrToStr(fold), reverse=True
)
# print(sorted_target_folder_list)
self.accumulate_unique_actions("pre_remove", var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"))
for folder_name in sorted_target_folder_list:
self.batch_accum += self.platform_helper.progress("Removing from {0}".format(folder_name))
var_stack.set_var("__TARGET_DIR__").append(os.path.normpath(folder_name))
items_in_folder = self.all_items_by_target_folder[folder_name]
self.batch_accum += self.platform_helper.new_line()
self.accumulate_unique_actions("pre_remove_from_folder", items_in_folder)
for IID in items_in_folder:
with self.install_definitions_index[IID].push_var_stack_scope() as installi:
self.batch_accum += self.platform_helper.progress("Removing {installi.name}...".format(**locals()))
for source_var in var_stack.get_configVar_obj("iid_source_var_list"):
source = var_stack.ResolveVarToList(source_var)
self.batch_accum += self.platform_helper.progress("Removing {source[0]}...".format(**locals()))
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_pre_remove_item", default=[])
self.create_remove_instructions_for_source(folder_name, source)
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_post_remove_item", default=[])
self.batch_accum += self.platform_helper.progress("Remove {source[0]} done".format(**locals()))
self.batch_accum += self.platform_helper.progress("Remove {installi.name} done".format(**locals()))
self.accumulate_unique_actions("post_remove_from_folder", items_in_folder)
self.batch_accum += self.platform_helper.progress("Remove from {0} done".format(folder_name))
self.accumulate_unique_actions("post_remove", var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"))
def __init__(self, initial_vars=None):
self.info_map_table = SVNTable()
self.items_table = IndexItemsTable()
ConfigVarYamlReader.__init__(self)
search_paths_var = var_stack.get_configVar_obj("__SEARCH_PATHS__")
self.path_searcher = utils.SearchPaths(search_paths_var)
self.url_translator = connectionBase.translate_url
self.init_default_vars(initial_vars)
# noinspection PyUnresolvedReferences
self.read_name_specific_defaults_file(super().__thisclass__.__name__)
# initialize the search paths helper with the current directory and dir where instl is now
self.path_searcher.add_search_path(os.getcwd())
self.path_searcher.add_search_path(os.path.dirname(os.path.realpath(sys.argv[0])))
self.path_searcher.add_search_path(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"))
self.platform_helper = PlatformSpecificHelperFactory(var_stack.ResolveVarToStr("__CURRENT_OS__"), self)
# init initial copy tool, tool might be later overridden after reading variable COPY_TOOL from yaml.
self.platform_helper.init_copy_tool()
self.install_definitions_index = dict()
self.batch_accum = BatchAccumulator()
self.do_not_write_vars = ("INFO_MAP_SIG", "INDEX_SIG", "PUBLIC_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "__CREDENTIALS__")
self.out_file_realpath = None
def create_copy_instructions(self):
self.create_sync_folder_manifest_command("before-copy")
# self.write_copy_debug_info()
# If we got here while in synccopy command, there is no need to read the info map again.
# If we got here while in copy command, read HAVE_INFO_MAP_FOR_COPY which defaults to HAVE_INFO_MAP_PATH.
# Copy might be called after the sync batch file was created
# but before it was executed in which case HAVE_INFO_MAP_FOR_COPY will be defined to NEW_HAVE_INFO_MAP_PATH.
if len(self.info_map_table.files_read_list) == 0:
have_info_path = var_stack.ResolveVarToStr("HAVE_INFO_MAP_FOR_COPY")
self.read_info_map_from_file(have_info_path)
# copy and actions instructions for sources
self.batch_accum.set_current_section('copy')
self.batch_accum += self.platform_helper.progress("Starting copy from $(COPY_SOURCES_ROOT_DIR)")
self.accumulate_unique_actions('pre_copy', var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"))
self.batch_accum += self.platform_helper.new_line()
sorted_target_folder_list = sorted(self.all_items_by_target_folder,
key=lambda fold: var_stack.ResolveStrToStr(fold))
# first create all target folders so to avoid dependency order problems such as creating links between folders
if len(sorted_target_folder_list) > 0:
self.batch_accum += self.platform_helper.progress("Create folders ...")
for target_folder_path in sorted_target_folder_list:
self.batch_accum += self.platform_helper.progress("Create folder {0} ...".format(target_folder_path))
if os.path.isfile(var_stack.ResolveStrToStr(target_folder_path)):
# weird as it maybe, some users have files where a folder should be.
# test for isfile is done here rather than in the batch file, because
# Windows does not have proper way to check "is file" in a batch.
self.batch_accum += self.platform_helper.rmfile(target_folder_path)
self.batch_accum += self.platform_helper.progress("Removed file that should be a folder {0}".format(target_folder_path))
self.batch_accum += self.platform_helper.mkdir_with_owner(target_folder_path)
self.batch_accum += self.platform_helper.progress("Create folder {0} done".format(target_folder_path))
self.batch_accum += self.platform_helper.progress("Create folders done")
if 'Mac' in var_stack.ResolveVarToList("__CURRENT_OS_NAMES__") and 'Mac' in var_stack.ResolveVarToList("TARGET_OS"):
self.pre_copy_mac_handling()
for target_folder_path in sorted_target_folder_list:
num_items_copied_to_folder = 0
items_in_folder = sorted(self.all_items_by_target_folder[target_folder_path])
self.batch_accum += self.platform_helper.new_line()
self.batch_accum += self.platform_helper.remark("- Begin folder {0}".format(target_folder_path))
self.batch_accum += self.platform_helper.cd(target_folder_path)
self.batch_accum += self.platform_helper.progress("copy to {0} ...".format(target_folder_path))
# accumulate pre_copy_to_folder actions from all items, eliminating duplicates
self.accumulate_unique_actions('pre_copy_to_folder', items_in_folder)
batch_accum_len_before = len(self.batch_accum)
self.batch_accum += self.platform_helper.copy_tool.begin_copy_folder()
for IID in items_in_folder:
with self.install_definitions_index[IID].push_var_stack_scope() as installi:
self.batch_accum += self.platform_helper.remark("-- Begin iid {0}".format(installi.iid))
for source_var in sorted(var_stack.get_configVar_obj("iid_source_var_list")):
source = var_stack.ResolveVarToList(source_var)
need_to_copy_source = installi.last_require_repo_rev == 0 or installi.last_require_repo_rev < self.get_max_repo_rev_for_source(source)
#print(installi.name, installi.last_require_repo_rev, need_to_copy_source, self.get_max_repo_rev_for_source(source))
if need_to_copy_source:
self.batch_accum += self.platform_helper.remark("--- Begin source {0}".format(source[0]))
num_items_copied_to_folder += 1
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_pre_copy_item", default=[])
self.create_copy_instructions_for_source(source, installi.name_and_version)
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_post_copy_item", default=[])
self.batch_accum += self.platform_helper.remark("--- End source {0}".format(source[0]))
self.batch_accum += self.platform_helper.remark("-- End iid {0}".format(installi.iid))
self.batch_accum += self.platform_helper.copy_tool.end_copy_folder()
# only if items were actually copied there's need to (Mac only) resolve symlinks
if num_items_copied_to_folder > 0:
if 'Mac' in var_stack.ResolveVarToList("__CURRENT_OS_NAMES__") and 'Mac' in var_stack.ResolveVarToList("TARGET_OS"):
self.batch_accum += self.platform_helper.progress("Resolve symlinks ...")
self.batch_accum += self.platform_helper.resolve_symlink_files()
self.batch_accum += self.platform_helper.progress("Resolve symlinks done")
# accumulate post_copy_to_folder actions from all items, eliminating duplicates
self.accumulate_unique_actions('post_copy_to_folder', items_in_folder)
self.batch_accum += self.platform_helper.progress("Copy to {0} done".format(target_folder_path))
#self.write_copy_to_folder_debug_info(target_folder_path)
self.batch_accum += self.platform_helper.remark("- End folder {0}".format(target_folder_path))
# actions instructions for sources that do not need copying, here folder_name is the sync folder
for sync_folder_name in sorted(self.no_copy_items_by_sync_folder.keys()):
items_in_folder = self.no_copy_items_by_sync_folder[sync_folder_name]
self.batch_accum += self.platform_helper.new_line()
self.batch_accum += self.platform_helper.cd(sync_folder_name)
self.batch_accum += self.platform_helper.progress("Actions in {0} ...".format(sync_folder_name))
# accumulate pre_copy_to_folder actions from all items, eliminating duplicates
self.accumulate_unique_actions('pre_copy_to_folder', items_in_folder)
for IID in sorted(items_in_folder):
with self.install_definitions_index[IID].push_var_stack_scope():
for source_var in sorted(var_stack.ResolveVarToList("iid_source_var_list", default=[])):
source = var_stack.ResolveVarToList(source_var)
source_folder, source_name = os.path.split(source[0])
to_unwtar = os.path.join(sync_folder_name, source_name)
self.batch_accum += self.platform_helper.unwtar_something(to_unwtar, no_artifacts=True)
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_pre_copy_item", default=[])
self.batch_accum += var_stack.ResolveVarToList("iid_action_list_post_copy_item", default=[])
# accumulate post_copy_to_folder actions from all items, eliminating duplicates
self.accumulate_unique_actions('post_copy_to_folder', items_in_folder)
self.batch_accum += self.platform_helper.progress("{sync_folder_name}".format(**locals()))
self.batch_accum += self.platform_helper.progress("Actions in {0} done".format(sync_folder_name))
print(self.bytes_to_copy, "bytes to copy")
self.accumulate_unique_actions('post_copy', var_stack.ResolveVarToList("__FULL_LIST_OF_INSTALL_TARGETS__"))
self.batch_accum.set_current_section('post-copy')
# Copy have_info file to "site" (e.g. /Library/Application support/... or c:\ProgramData\...)
# for reference. But when preparing offline installers the site location is the same as the sync location
# so copy should be avoided.
if var_stack.ResolveVarToStr("HAVE_INFO_MAP_PATH") != var_stack.ResolveVarToStr("SITE_HAVE_INFO_MAP_PATH"):
self.batch_accum += self.platform_helper.mkdir_with_owner("$(SITE_REPO_BOOKKEEPING_DIR)")
self.batch_accum += self.platform_helper.progress("Created folder $(SITE_REPO_BOOKKEEPING_DIR)")
self.batch_accum += self.platform_helper.copy_file_to_file("$(HAVE_INFO_MAP_PATH)", "$(SITE_HAVE_INFO_MAP_PATH)")
self.batch_accum += self.platform_helper.progress("Copied $(HAVE_INFO_MAP_PATH) to $(SITE_HAVE_INFO_MAP_PATH)")
self.platform_helper.copy_tool.finalize()
self.create_require_file_instructions()
# messages about orphan iids
for iid in sorted(var_stack.ResolveVarToList("__ORPHAN_INSTALL_TARGETS__")):
self.batch_accum += self.platform_helper.echo("Don't know how to install " + iid)
self.batch_accum += self.platform_helper.progress("Done copy")