def read_include_node(self, i_node, *args, **kwargs):
if i_node.isScalar():
kwargs['original-path-to-file'] = i_node.value
resolved_file_name = config_vars.resolve_str(i_node.value)
self.read_yaml_file(resolved_file_name, *args, **kwargs)
elif i_node.isSequence():
for sub_i_node in i_node:
self.read_include_node(sub_i_node, *args, **kwargs)
elif i_node.isMapping():
if "url" in i_node:
file_was_downloaded_and_read = False
kwargs['original-path-to-file'] = i_node["url"].value
resolved_file_url = config_vars.resolve_str(
i_node["url"].value)
expected_checksum = None
if "checksum" in i_node:
expected_checksum = config_vars.resolve_str(
i_node["checksum"].value)
try:
file_path = utils.download_from_file_or_url(
in_url=resolved_file_url,
config_vars=config_vars,
in_target_path=None,
translate_url_callback=connectionBase.translate_url,
cache_folder=self.get_aux_cache_dir(make_dir=True),
expected_checksum=expected_checksum)
self.read_yaml_file(file_path, *args, **kwargs)
file_was_downloaded_and_read = True
except (FileNotFoundError, urllib.error.URLError):
ignore = kwargs.get('ignore_if_not_exist', False)
if ignore:
self.progress(
f"'ignore_if_not_exist' specified, ignoring FileNotFoundError for {resolved_file_url}"
)
else:
raise
if "copy" in i_node and file_was_downloaded_and_read:
self.batch_accum.set_current_section('post')
for copy_destination in i_node["copy"]:
need_to_copy = True
destination_file_resolved_path = utils.ExpandAndResolvePath(
config_vars.resolve_str(copy_destination.value))
if destination_file_resolved_path.is_file(
) and expected_checksum is not None:
checksums_match = utils.check_file_checksum(
file_path=destination_file_resolved_path,
expected_checksum=expected_checksum)
need_to_copy = not checksums_match
if need_to_copy:
self.batch_accum += MakeDir(
destination_file_resolved_path.parent,
chowner=True)
self.batch_accum += CopyFileToFile(
file_path,
destination_file_resolved_path,
hard_links=False,
copy_owner=True)
def create_create_folders_instructions(self, folder_list: List[str]) -> None:
with self.batch_accum.sub_accum(Stage("create folders")) as create_folders_section:
kwargs_defaults = {'remove_obstacles': True, 'chowner': False, 'recursive_chmod': False}
third_party_folders = [utils.ExpandAndResolvePath(config_vars.resolve_str(os.fspath(p))) for p in config_vars.get("THIRD_PARTY_FOLDERS", []).list()]
for target_folder_path in folder_list:
target_folder_path = utils.ExpandAndResolvePath(config_vars.resolve_str(os.fspath(target_folder_path)))
our_folder = next((False for p in third_party_folders if p == target_folder_path), True)
create_folders_section += MakeDir(target_folder_path, chowner=our_folder, recursive_chmod=False)
def create_remove_instructions(self):
have_info_path = config_vars["HAVE_INFO_MAP_PATH"].Path()
if not have_info_path or not have_info_path.is_file():
have_info_path = config_vars["SITE_HAVE_INFO_MAP_PATH"].Path()
self.info_map_table.read_from_file(have_info_path,
disable_indexes_during_read=True)
self.calc_iid_to_name_and_version()
self.batch_accum.set_current_section('remove')
self.batch_accum += Progress("Start remove")
sorted_target_folder_list = sorted(
self.all_iids_by_target_folder,
key=lambda fold: config_vars.resolve_str(fold),
reverse=True)
self.batch_accum += self.accumulate_unique_actions_for_active_iids(
'pre_remove',
list(config_vars["__FULL_LIST_OF_INSTALL_TARGETS__"]))
for folder_name in sorted_target_folder_list:
with self.batch_accum.sub_accum(
Stage("Remove from folder",
folder_name)) as folder_accum_transaction:
folder_accum_transaction += self.create_remove_previous_sources_instructions_for_target_folder(
folder_name)
config_vars["__TARGET_DIR__"] = Path(folder_name)
items_in_folder = self.all_iids_by_target_folder[folder_name]
folder_accum_transaction += self.accumulate_unique_actions_for_active_iids(
'pre_remove_from_folder', items_in_folder)
for IID in items_in_folder:
name_for_iid = self.name_for_iid(iid=IID)
with folder_accum_transaction.sub_accum(
Stage("Remove",
name_for_iid)) as iid_accum_transaction:
sources_for_iid = self.items_table.get_sources_for_iid(
IID)
resolved_sources_for_iid = [
(config_vars.resolve_str(s[0]), s[1])
for s in sources_for_iid
]
for source in resolved_sources_for_iid:
_, source_leaf = os.path.split(source[0])
iid_accum_transaction += self.accumulate_actions_for_iid(
iid=IID, detail_name="pre_remove_item")
iid_accum_transaction += self.create_remove_instructions_for_source(
IID, folder_name, source)
iid_accum_transaction += self.accumulate_actions_for_iid(
iid=IID, detail_name="post_remove_item")
folder_accum_transaction += self.accumulate_unique_actions_for_active_iids(
'post_remove_from_folder', items_in_folder)
self.batch_accum += self.accumulate_unique_actions_for_active_iids(
'post_remove',
list(config_vars["__FULL_LIST_OF_INSTALL_TARGETS__"]))
def test_array(self):
config_vars["PUSHKIN"] = "1", "2", "3"
self.assertEqual("123", config_vars["PUSHKIN"].str())
self.assertEqual("123", config_vars.resolve_str("$(PUSHKIN)"))
self.assertEqual("1", config_vars.resolve_str("$(PUSHKIN[0])"))
self.assertEqual("2", config_vars.resolve_str("$(PUSHKIN[1])"))
self.assertEqual("3", config_vars.resolve_str("$(PUSHKIN[2])"))
self.assertEqual(
"321",
config_vars.resolve_str("$(PUSHKIN[2])$(PUSHKIN[1])$(PUSHKIN[0])"))
def __call__(self, *args, **kwargs) -> None:
# fill the iid_to_svn_item_t table
self.info_map_table.populate_IIDToSVNItem()
# get the list of info map file names
info_map_to_item = dict()
all_info_map_names = self.items_table.get_unique_detail_values('info_map')
for infomap_file_name in all_info_map_names:
info_map_file_path = self.work_folder.joinpath(infomap_file_name)
if info_map_file_path.is_file():
log.info(f"{infomap_file_name} was found so no need to create it")
# file already exists, probably copied from the "Common" repository
# just checking that the fie is also zipped
zip_infomap_file_name = config_vars.resolve_str(infomap_file_name+"$(WZLIB_EXTENSION)")
zip_info_map_file_path = self.work_folder.joinpath(zip_infomap_file_name)
if not zip_info_map_file_path.is_file():
raise FileNotFoundError(f"found {info_map_file_path} but not {zip_info_map_file_path}")
else:
self.info_map_table.mark_items_required_by_infomap(infomap_file_name)
info_map_items = self.info_map_table.get_required_items()
info_map_to_item[infomap_file_name] = info_map_items
files_to_add_to_default_info_map = list() # the named info_map files and their wzip version should be added to the default info_map
# write each info map to file
for infomap_file_name, info_map_items in info_map_to_item.items():
if info_map_items: # could be that no items are linked to the info map file
info_map_file_path = self.work_folder.joinpath(infomap_file_name)
self.info_map_table.write_to_file(in_file=info_map_file_path, items_list=info_map_items, field_to_write=self.fields_relevant_to_info_map)
files_to_add_to_default_info_map.append(info_map_file_path)
zip_infomap_file_name = config_vars.resolve_str(infomap_file_name+"$(WZLIB_EXTENSION)")
zip_info_map_file_path = self.work_folder.joinpath(zip_infomap_file_name)
with Wzip(info_map_file_path, self.work_folder, own_progress_count=0) as wzipper:
wzipper()
files_to_add_to_default_info_map.append(zip_info_map_file_path)
# add the default info map
default_info_map_file_name = str(config_vars["MAIN_INFO_MAP_FILE_NAME"])
default_info_map_file_path = self.work_folder.joinpath(default_info_map_file_name)
info_map_items = self.info_map_table.get_items_for_default_infomap()
self.info_map_table.write_to_file(in_file=default_info_map_file_path, items_list=info_map_items, field_to_write=self.fields_relevant_to_info_map)
with Wzip(default_info_map_file_path, self.work_folder, own_progress_count=0) as wzipper:
wzipper()
# add a line to default info map for each non default info_map created above
with utils.utf8_open_for_read(default_info_map_file_path, "a") as wfd:
for file_to_add in files_to_add_to_default_info_map:
file_checksum = utils.get_file_checksum(file_to_add)
file_size = file_to_add.stat().st_size
# todo: make path relative
line_for_main_info_map = f"instl/{file_to_add.name}, f, {config_vars['TARGET_REPO_REV'].str()}, {file_checksum}, {file_size}\n"
wfd.write(line_for_main_info_map)
def activate_repo_rev(self):
try:
if self.redis_conn:
current_items = self.tree.get_children()
domain_repo = self.tk_vars["DOMAIN_REPO_TO_ACTIVATE"].get()
if domain_repo in current_items:
host = self.redis_conn.host
repo_rev = self.tk_vars["REPO_REV_TO_ACTIVATE"].get()
redis_value = config_vars.resolve_str(":".join(('activate', domain_repo, str(repo_rev))))
redis_key = config_vars.resolve_str(":".join(("$(REDIS_KEYS_PREFIX)", host, "waiting_list")))
answer = messagebox.askyesno("Activate repo-rev", f"Activate repo-rev {repo_rev} on {domain_repo} ?")
if answer:
self.redis_conn.lpush(redis_key, redis_value)
except Exception as ex:
print(f"activate_repo_rev exception {ex}")
def test_Plist_for_native_instruments(self):
config_vars[
"Plist_for_native_instruments"] = r'''ShellCommand('"$(LOCAL_REPO_SYNC_DIR)/Mac/Utilities/plist/plist_creator.sh" $(__Plist_for_native_instruments_1__) $(__Plist_for_native_instruments_2__)', ignore_all_errors=True)'''
o = config_vars.resolve_str(
'$(Plist_for_native_instruments<"Aphex Vintage Exciter", "/Applications/Waves/Data/NKS FX/">)'
)
print(o)
def init_default_client_vars(self):
if "SYNC_BASE_URL" in config_vars:
resolved_sync_base_url = config_vars["SYNC_BASE_URL"].str()
url_main_item = utils.main_url_item(resolved_sync_base_url)
config_vars["SYNC_BASE_URL_MAIN_ITEM"] = url_main_item
# TARGET_OS_NAMES defaults to __CURRENT_OS_NAMES__, which is not what we want if syncing to
# an OS which is not the current
if config_vars["TARGET_OS"].str() != config_vars["__CURRENT_OS__"].str(
):
target_os_names = list(config_vars[config_vars.resolve_str(
"$(TARGET_OS)_ALL_OS_NAMES")])
config_vars["TARGET_OS_NAMES"] = target_os_names
second_name: str = config_vars["TARGET_OS"].str()
if len(target_os_names) > 1:
second_name = target_os_names[1]
config_vars["TARGET_OS_SECOND_NAME"] = second_name
if "REPO_TYPE" in config_vars: # some commands do not need to have REPO_TYPE
self.read_defaults_file(str(config_vars["REPO_TYPE"]))
# AUXILIARY_IIDS are iids that are not real products such as UNINSTALL_AS_... iids
if "AUXILIARY_IIDS" not in config_vars:
log.warning(f"could not find configVar 'AUXILIARY_IIDS'")
self.auxiliary_iids.extend(list(config_vars.get("AUXILIARY_IIDS", [])))
config_vars["__MAIN_DRIVE_NAME__"] = utils.get_main_drive_name()
def create_copy_instructions(self) -> None:
self.progress("create copy instructions ...")
# If we got here while in synccopy command, there is no need to read the info map again.
# If we got here while in copy command, read HAVE_INFO_MAP_COPY_PATH which defaults to NEW_HAVE_INFO_MAP_PATH.
# Copy might be called after the sync batch file was created but before it was executed
if len(self.info_map_table.files_read_list) == 0:
have_info_path = os.fspath(config_vars["HAVE_INFO_MAP_COPY_PATH"])
self.info_map_table.read_from_file(have_info_path, disable_indexes_during_read=True)
self.avoid_copy_markers = list(config_vars.get('AVOID_COPY_MARKERS', []))
# copy and actions instructions for sources
self.batch_accum.set_current_section('copy')
self.batch_accum += self.create_sync_folder_manifest_command("before-copy", back_ground=True)
self.batch_accum += Progress("Start copy from $(COPY_SOURCES_ROOT_DIR)")
sorted_target_folder_list = sorted(self.all_iids_by_target_folder,
key=lambda fold: config_vars.resolve_str(fold))
# first create all target folders so to avoid dependency order problems such as creating links between folders
self.create_create_folders_instructions(sorted_target_folder_list)
self.batch_accum += self.accumulate_unique_actions_for_active_iids('pre_copy')
if self.mac_current_and_target:
self.pre_copy_mac_handling()
remove_previous_sources = bool(config_vars.get("REMOVE_PREVIOUS_SOURCES",True))
for target_folder_path in sorted_target_folder_list:
if remove_previous_sources:
with self.batch_accum.sub_accum(Stage("remove_previous_sources_instructions_for_target_folder", target_folder_path)) as seb_sec:
seb_sec += self.create_remove_previous_sources_instructions_for_target_folder(target_folder_path)
self.create_copy_instructions_for_target_folder(target_folder_path)
# actions instructions for sources that do not need copying, here folder_name is the sync folder
for sync_folder_name in sorted(self.no_copy_iids_by_sync_folder.keys()):
with self.batch_accum.sub_accum(CdStage("create_copy_instructions_for_no_copy_folder", sync_folder_name)) as folder_accum:
folder_accum += self.create_copy_instructions_for_no_copy_folder(sync_folder_name)
self.progress(self.bytes_to_copy, "bytes to copy")
self.batch_accum += self.accumulate_unique_actions_for_active_iids('post_copy')
self.batch_accum.set_current_section('post-copy')
# Copy have_info file to "site" (e.g. /Library/Application support/... or c:\ProgramData\...)
# for reference. But when preparing offline installers the site location is the same as the sync location
# so copy should be avoided.
if os.fspath(config_vars["HAVE_INFO_MAP_PATH"]) != os.fspath(config_vars["SITE_HAVE_INFO_MAP_PATH"]):
self.batch_accum += MakeDir("$(SITE_REPO_BOOKKEEPING_DIR)", chowner=True)
self.batch_accum += CopyFileToFile("$(HAVE_INFO_MAP_PATH)", "$(SITE_HAVE_INFO_MAP_PATH)", hard_links=False, copy_owner=True)
self.create_require_file_instructions()
# messages about orphan iids
for iid in sorted(list(config_vars["__ORPHAN_INSTALL_TARGETS__"])):
self.batch_accum += Echo(f"Don't know how to install {iid}")
self.batch_accum += Progress("Done copy")
self.progress("create copy instructions done")
self.progress("")
def create_copy_instructions_for_target_folder(self,
target_folder_path) -> None:
with self.batch_accum.sub_accum(
CdStage("copy_to_folder",
target_folder_path)) as copy_to_folder_accum:
self.current_destination_folder = target_folder_path
num_items_copied_to_folder = 0
items_in_folder = sorted(
self.all_iids_by_target_folder[target_folder_path])
# accumulate pre_copy_to_folder actions from all items, eliminating duplicates
copy_to_folder_accum += self.accumulate_unique_actions_for_active_iids(
'pre_copy_to_folder', items_in_folder)
num_symlink_items: int = 0
for IID in items_in_folder:
name_and_version = self.name_and_version_for_iid(iid=IID)
with copy_to_folder_accum.sub_accum(
Stage("copy", name_and_version)) as iid_accum:
self.current_iid = IID
sources_for_iid = self.items_table.get_sources_for_iid(IID)
resolved_sources_for_iid = [(config_vars.resolve_str(s[0]),
s[1])
for s in sources_for_iid]
for source in resolved_sources_for_iid:
self.progress(
f"create copy instructions of {source[0]} to {config_vars.resolve_str(target_folder_path)}"
)
with iid_accum.sub_accum(
Stage("copy source",
source[0])) as source_accum:
num_items_copied_to_folder += 1
source_accum += self.accumulate_actions_for_iid(
iid=IID,
detail_name="pre_copy_item",
message=None)
source_accum += self.create_copy_instructions_for_source(
source, name_and_version)
source_accum += self.accumulate_actions_for_iid(
iid=IID,
detail_name="post_copy_item",
message=None)
if self.mac_current_and_target:
num_symlink_items += self.info_map_table.count_symlinks_in_dir(
source[0])
self.current_iid = None
# only if items were actually copied there's need to (Mac only) resolve symlinks
if self.mac_current_and_target:
if num_items_copied_to_folder > 0 and num_symlink_items > 0:
copy_to_folder_accum += ResolveSymlinkFilesInFolder(
target_folder_path,
own_progress_count=num_symlink_items)
# accumulate post_copy_to_folder actions from all items, eliminating duplicates
if copy_to_folder_accum.is_essential():
copy_to_folder_accum += self.accumulate_unique_actions_for_active_iids(
'post_copy_to_folder', items_in_folder)
self.current_destination_folder = None
def pre_resolve_path(self, path_to_resolve) -> str:
""" for some paths we cannot wait for resolution in the batch file"""
resolved_path = config_vars.resolve_str(path_to_resolve)
try:
resolved_path = str(Path(resolved_path).resolve())
except:
pass
return resolved_path
def get_direct_sync_status_from_indicator(self, direct_sync_indicator):
retVal = False
if direct_sync_indicator is not None:
try:
retVal = utils.str_to_bool_int(config_vars.resolve_str(direct_sync_indicator))
except:
pass
return retVal
def get_version_str(self, short=False):
instl_ver_str = ".".join(list(config_vars["__INSTL_VERSION__"]))
if not short:
if "__PLATFORM_NODE__" not in config_vars:
config_vars.update({"__PLATFORM_NODE__": platform.node()})
instl_ver_str = config_vars.resolve_str(
"$(INSTL_EXEC_DISPLAY_NAME) version " + instl_ver_str +
" $(__COMPILATION_TIME__) $(__PLATFORM_NODE__)")
return instl_ver_str
def __call__(self, *args, **kwargs) -> None:
pybatch.PythonBatchCommandBase.__call__(self, *args, **kwargs)
if self.config_files is not None:
reader = ConfigVarYamlReader(config_vars)
for config_file in self.config_files:
reader.read_yaml_file(config_file)
with utils.utf8_open_for_read(self.unresolved_file, "r") as rfd:
text_to_resolve = rfd.read()
resolved_text = config_vars.resolve_str(text_to_resolve)
with utils.utf8_open_for_write(self.resolved_file, "w") as wfd:
wfd.write(resolved_text)
def prepare_command_list_from_file(self):
command_lines = list()
for config_file in config_vars["__CONFIG_FILE__"].list():
with utils.utf8_open_for_read(os.fspath(config_file), "r") as rfd:
command_lines.extend(rfd.readlines())
command_list = list()
for command_line in command_lines:
resolved_command_line = config_vars.resolve_str(
command_line.strip())
argv = shlex.split(resolved_command_line)
command_list.append(argv)
return command_list
def create_remove_previous_sources_instructions_for_target_folder(self, target_folder_path):
retVal = AnonymousAccum()
target_folder_path_resolved = utils.ExpandAndResolvePath(config_vars.resolve_str(target_folder_path))
if target_folder_path_resolved.is_dir(): # no need to remove previous sources if folder does not exist
iids_in_folder = self.all_iids_by_target_folder[target_folder_path]
previous_sources = self.items_table.get_details_and_tag_for_active_iids("previous_sources", unique_values=True, limit_to_iids=iids_in_folder)
if len(previous_sources) > 0:
with retVal.sub_accum(Cd(target_folder_path)) as remove_prev_section:
remove_prev_section += Progress(f"remove previous versions {target_folder_path}")
for previous_source in previous_sources:
remove_prev_section += self.create_remove_previous_sources_instructions_for_source(target_folder_path, previous_source)
return retVal
def read_previous_requirements(self):
require_file_path = config_vars["SITE_REQUIRE_FILE_PATH"].Path()
with Chmod(require_file_path, "a+rw", ignore_all_errors=True) as chmoder:
chmoder()
try:
self.read_yaml_file(require_file_path, ignore_if_not_exist=True)
except Exception as ex:
log.warning(f"Exception reading {require_file_path}: {ex}")
renamed_require_file_path = Path(require_file_path.parent, config_vars.resolve_str("$(SITE_REQUIRE_FILE_NAME).failed_to_read"))
try:
require_file_path.rename(renamed_require_file_path)
log.warning(f"moved require.yaml to {renamed_require_file_path}")
except Exception as ex_in_ex:
log.warning(f"failed to moved require.yaml to {renamed_require_file_path}: {ex_in_ex}")
def get_default_db_file(self):
if "__MAIN_DB_FILE__" not in config_vars:
db_base_path = None
if "__MAIN_OUT_FILE__" in config_vars:
# try to set the db file next to the output file
db_base_path = config_vars["__MAIN_OUT_FILE__"].Path()
elif "__MAIN_INPUT_FILE__" in config_vars:
# if no output file try next to the input file
db_base_path = Path(
config_vars.resolve_str(
"$(__MAIN_INPUT_FILE__)-$(__MAIN_COMMAND__)"))
else:
# as last resort try the Logs folder
logs_dir = utils.get_system_log_folder_path()
if logs_dir.is_dir():
db_base_path = logs_dir.joinpath(
config_vars.resolve_str("instl-$(__MAIN_COMMAND__)"))
if db_base_path:
# set the proper extension
db_base_path = db_base_path.parent.joinpath(
db_base_path.name +
config_vars.resolve_str(".$(DB_FILE_EXT)"))
config_vars["__MAIN_DB_FILE__"] = db_base_path
log.info(f'DB FILE: {config_vars["__MAIN_DB_FILE__"].str()}')
if self._owner.refresh_db_file:
if config_vars["__MAIN_DB_FILE__"].str() != ":memory:":
db_base_path = config_vars["__MAIN_DB_FILE__"].Path()
if db_base_path.is_file():
utils.safe_remove_file(db_base_path)
log.info(
f'DB FILE REMOVED: {config_vars["__MAIN_DB_FILE__"].str()}'
)
else:
log.info(
f'DB FILE DOES NOT EXIST: {config_vars["__MAIN_DB_FILE__"].str()}'
)
def get_default_db_file(self):
if "__MAIN_DB_FILE__" not in config_vars:
db_base_path = None
if "__MAIN_OUT_FILE__" in config_vars:
# try to set the db file next to the output file
db_base_path = config_vars["__MAIN_OUT_FILE__"].Path()
elif "__MAIN_INPUT_FILE__" in config_vars:
# if no output file try next to the input file
db_base_path = Path(config_vars.resolve_str("$(__MAIN_INPUT_FILE__)-$(__MAIN_COMMAND__)"))
else:
# as last resort try the Logs folder on desktop if one exists
logs_dir = Path(os.path.expanduser("~"), "Desktop", "Logs")
if logs_dir.is_dir():
db_base_path = logs_dir.joinpath(config_vars.resolve_str("instl-$(__MAIN_COMMAND__)"))
if db_base_path:
# set the proper extension
db_base_path = db_base_path.parent.joinpath(db_base_path.name+config_vars.resolve_str(".$(DB_FILE_EXT)"))
config_vars["__MAIN_DB_FILE__"] = db_base_path
if self._owner.refresh_db_file:
db_base_path = config_vars["__MAIN_DB_FILE__"].Path()
if db_base_path.is_file():
utils.safe_remove_file(db_base_path)
def update_state(self, *args, **kwargs): # ClientFrameController
super().update_state(*args, **kwargs)
self.update_client_input_file_combo()
input_file_base_name = config_vars["CLIENT_GUI_IN_FILE"].Path().name
config_vars["CLIENT_GUI_IN_FILE_NAME"] = input_file_base_name
if self.tk_vars["CLIENT_GUI_CMD"].get() in list(config_vars["__COMMANDS_WITH_RUN_OPTION__"]):
self.client_run_batch_file_checkbox.configure(state='normal')
else:
self.client_run_batch_file_checkbox.configure(state='disabled')
command_line = " ".join(self.create_client_command_line())
self.text_widget.configure(state='normal')
self.text_widget.delete(1.0, END)
self.text_widget.insert(END, config_vars.resolve_str(command_line))
self.text_widget.configure(state='disabled')
def should_copy_source(self, source, target_folder_path):
retVal = True
reason_not_to_copy = None
if not self.update_mode:
top_src = config_vars["COPY_SOURCES_ROOT_DIR"].Path(
resolve=True).joinpath(source[0])
top_trg = Path(config_vars.resolve_str(target_folder_path),
top_src.name)
if top_trg.exists():
if source[1] == "!dir":
trg = top_trg.joinpath("Contents")
if trg.exists():
# try to Info.xml or Info.plist under contents
src = top_src.joinpath("Contents")
for avoid_copy_marker in self.avoid_copy_markers:
src_marker = src.joinpath(avoid_copy_marker)
dst_marker = trg.joinpath(avoid_copy_marker)
same_checksums = utils.compare_files_by_checksum(
dst_marker, src_marker)
if same_checksums:
reason_not_to_copy = f"same checksum Contents/{avoid_copy_marker}"
retVal = False
break
else:
# try to Info.xml or Info.plist at top level
for avoid_copy_marker in self.avoid_copy_markers:
src_marker = top_src.joinpath(avoid_copy_marker)
dst_marker = top_trg.joinpath(avoid_copy_marker)
same_checksums = utils.compare_files_by_checksum(
dst_marker, src_marker)
if same_checksums:
reason_not_to_copy = f"same checksum {avoid_copy_marker} in top level"
retVal = False
break
elif source[1] == "!file":
try:
if top_src.stat().st_ino == top_trg.stat().st_ino:
retVal = False
reason_not_to_copy = f"same inode"
except:
pass
return retVal, reason_not_to_copy
def handle_yaml_read_error(self, **kwargs):
try:
the_node_stack = kwargs.get('node-stack', "unknown")
position_in_file = getattr(the_node_stack, "start_mark", "unknown")
original_path_to_file = utils.ExpandAndResolvePath(
config_vars.resolve_str(kwargs.get('original-path-to-file',
'')))
yaml_read_errors = list()
yaml_read_errors.append("yaml_read_error:")
if os.fspath(original_path_to_file) not in position_in_file:
yaml_read_errors.append(
f""" path-to-file: {original_path_to_file}""")
yaml_read_errors.append(
f""" position-in-file: {position_in_file}""")
yaml_read_errors.append(
f""" permissions: {utils.single_disk_item_listing(original_path_to_file)}"""
)
yaml_read_errors.append(
f""" exception: {kwargs.get('exception', '')}""")
log.error("\n".join(yaml_read_errors))
except Exception as ex:
pass
def update_state(self, *args, **kwargs): # AdminFrameController
super().update_state(*args, **kwargs)
self.read_admin_config_files()
input_file_base_name = Path(config_vars["ADMIN_GUI_LOCAL_CONFIG_FILE"].raw()).name
config_vars["ADMIN_GUI_CONFIG_FILE_NAME"] = input_file_base_name
if self.tk_vars["ADMIN_GUI_CMD"].get() in list(config_vars["__COMMANDS_WITH_LIMIT_OPTION__"]):
self.limit_path_entry_widget.configure(state='normal')
else:
self.limit_path_entry_widget.configure(state='disabled')
if self.tk_vars["ADMIN_GUI_CMD"].get() in list(config_vars["__COMMANDS_WITH_RUN_OPTION__"]):
self.admin_run_batch_file_checkbox.configure(state='normal')
else:
self.admin_run_batch_file_checkbox.configure(state='disabled')
command_line = " ".join([shlex.quote(p) for p in self.create_admin_command_line()])
self.text_widget.configure(state='normal')
self.text_widget.delete(1.0, END)
self.text_widget.insert(END, config_vars.resolve_str(command_line))
self.text_widget.configure(state='disabled')
def __call__(self, *args, **kwargs) -> None:
pybatch.PythonBatchCommandBase.__call__(self, *args, **kwargs)
with config_vars.push_scope_context() as scope_context:
if self.temp_config_vars:
config_vars.update(self.temp_config_vars)
if self.config_files is not None:
reader = ConfigVarYamlReader(config_vars)
for config_file in self.config_files:
reader.read_yaml_file(config_file)
with utils.utf8_open_for_read(self.unresolved_file, "r") as rfd:
text_to_resolve = rfd.read()
resolved_text = config_vars.resolve_str(text_to_resolve)
if self.raise_if_unresolved:
unresolved_re = re.compile(r"""\$\(.*?\)""")
all_unresolved = unresolved_re.findall(resolved_text)
if all_unresolved:
unresolved_references = ", ".join(list(
set(all_unresolved)))
raise ValueError(
f"unresolved config_vars in {self.unresolved_file}: {unresolved_references}"
)
with utils.utf8_open_for_write(self.resolved_file, "w") as wfd:
wfd.write(resolved_text)
def __repr__(self):
single_indent = " "
running_progress_count = self.initial_progress
PythonBatchCommandBase.config_vars_for_repr = config_vars # so __repr__ of object derived from PythonBatchCommandBase will resolve config_vars values
def _create_unique_obj_name(obj, prog_count):
try:
_create_unique_obj_name.instance_counter += 1
except AttributeError:
_create_unique_obj_name.instance_counter = 1
obj_name = camel_to_snake_case(
f"{obj.__class__.__name__}_{_create_unique_obj_name.instance_counter:03}_{prog_count}"
)
return obj_name
def _remark_helper(*the_remarks):
retVal = ", ".join(
str(remark) for remark in filter(None, the_remarks))
if retVal:
retVal = f""" # {retVal}"""
return retVal
def _repr_helper(batch_items, io_str, indent):
nonlocal running_progress_count
indent_str = single_indent * indent
if isinstance(batch_items, list):
for item in batch_items:
_repr_helper(item, io_str, indent)
else:
running_progress_count += batch_items.own_progress_count
batch_items.prog_num = running_progress_count
if batch_items.call__call__ is False and batch_items.is_context_manager is False:
text_to_write = f"""{indent_str}{repr(batch_items)}\n"""
io_str.write(text_to_write)
_repr_helper(batch_items.child_batch_commands, io_str,
indent)
elif batch_items.call__call__ is False and batch_items.is_context_manager is True:
text_to_write = f"""{indent_str}with {repr(batch_items)}:\n"""
io_str.write(text_to_write)
if batch_items.child_batch_commands:
_repr_helper(batch_items.child_batch_commands, io_str,
indent + 1)
else:
text_to_write = f"""{indent_str}{single_indent}pass\n"""
io_str.write(text_to_write)
elif batch_items.call__call__ is True and batch_items.is_context_manager is False:
text_to_write = f"""{indent_str}{repr(batch_items)}()\n"""
io_str.write(text_to_write)
_repr_helper(batch_items.child_batch_commands, io_str,
indent)
elif batch_items.call__call__ is True and batch_items.is_context_manager is True:
obj_name = _create_unique_obj_name(batch_items,
running_progress_count)
text_to_write = f"""{indent_str}with {repr(batch_items)} as {obj_name}:\n"""
io_str.write(text_to_write)
text_to_write = f"""{indent_str}{single_indent}{obj_name}("""
text_to_write += ")\n"
io_str.write(text_to_write)
_repr_helper(batch_items.child_batch_commands, io_str,
indent + 1)
self.set_current_section('epilog')
self += PatchPyBatchWithTimings(config_vars['__MAIN_OUT_FILE__'])
PythonBatchCommandBase.total_progress = 0
for name, section in self.sections.items():
progress_count_for_section = section.total_progress_count()
PythonBatchCommandBase.total_progress += progress_count_for_section
PythonBatchCommandBase.total_progress += 1 # count the PythonBatchRuntime, todo: a better way to add PythonBatchRuntime's progress count to the total
prolog_str = io.StringIO()
prolog_str.write(self._python_opening_code())
if 'assign' in self.sections:
_repr_helper(self.sections['assign'], prolog_str, 0)
main_str = io.StringIO()
the_command = config_vars.get("__MAIN_COMMAND__", "woolly mammoth")
runtimer = PythonBatchRuntime(the_command)
for section_name in PythonBatchCommandAccum.section_order:
if section_name in self.sections:
if section_name not in PythonBatchCommandAccum.special_sections:
runtimer += self.sections[section_name]
main_str.write("\n")
_repr_helper(runtimer, main_str, 0)
epilog_str = io.StringIO()
if 'epilog' in self.sections:
main_str.write("\n")
_repr_helper(self.sections['epilog'], epilog_str, 0)
epilog_str.write(self._python_closing_code())
main_str_resolved = config_vars.resolve_str(main_str.getvalue())
main_str_resolved = config_vars.replace_unresolved_with_native_var_pattern(
main_str_resolved,
list(config_vars["__CURRENT_OS_NAMES__"])[0])
the_whole_repr = prolog_str.getvalue(
) + main_str_resolved + epilog_str.getvalue()
PythonBatchCommandBase.config_vars_for_repr = None
return the_whole_repr
def do_resolve(self, param):
if param:
print(config_vars.resolve_str(param))
return False
def test_var_in_var_simple(self):
config_vars["A"] = "$(B)"
config_vars["B"] = "$(C)"
config_vars["C"] = "ali baba"
self.assertEqual("ali baba", config_vars["A"].str())
self.assertEqual("ali baba", config_vars.resolve_str("$(A)"))
def read_remote_info_map(self):
""" Reads the info map of the static files available for syncing.
Writes the map to local sync folder for reference and debugging.
"""
info_map_file_url = None
try:
with self.instlObj.info_map_table.reading_files_context():
os.makedirs(os.fspath(
config_vars["LOCAL_REPO_BOOKKEEPING_DIR"]),
exist_ok=True)
os.makedirs(os.fspath(
config_vars["LOCAL_REPO_REV_BOOKKEEPING_DIR"]),
exist_ok=True)
if "INSTL_FOLDER_BASE_URL" not in config_vars:
if "REPO_REV_FOLDER_HIERARCHY" not in config_vars:
config_vars[
"REPO_REV_FOLDER_HIERARCHY"] = self.instlObj.repo_rev_to_folder_hierarchy(
config_vars["REPO_REV"].str())
config_vars[
"INSTL_FOLDER_BASE_URL"] = "$(BASE_LINKS_URL)/$(REPO_NAME)/$(REPO_REV_FOLDER_HIERARCHY)/instl"
if "INFO_MAP_FILE_URL" not in config_vars:
config_vars["INFO_MAP_FILE_URL"] = config_vars.resolve_str(
"$(INSTL_FOLDER_BASE_URL)/info_map.txt")
info_map_file_url = config_vars["INFO_MAP_FILE_URL"].str()
info_map_file_expected_checksum = None
if "INFO_MAP_CHECKSUM" in config_vars:
info_map_file_expected_checksum = config_vars[
"INFO_MAP_CHECKSUM"].str()
local_copy_of_info_map_in = os.fspath(
config_vars["LOCAL_COPY_OF_REMOTE_INFO_MAP_PATH"])
local_copy_of_info_map_out = utils.download_from_file_or_url(
in_url=info_map_file_url,
config_vars=config_vars,
in_target_path=local_copy_of_info_map_in,
translate_url_callback=connectionBase.translate_url,
cache_folder=self.instlObj.get_default_sync_dir(
continue_dir="cache", make_dir=True),
expected_checksum=info_map_file_expected_checksum)
self.instlObj.info_map_table.read_from_file(
local_copy_of_info_map_out)
self.instlObj.progress(f"read info_map {info_map_file_url}")
additional_info_maps = self.instlObj.items_table.get_details_for_active_iids(
"info_map", unique_values=True)
for additional_info_map in additional_info_maps:
# try to get the zipped info_map
additional_info_map_file_name = config_vars.resolve_str(
f"{additional_info_map}$(WZLIB_EXTENSION)")
path_in_main_info_map = config_vars.resolve_str(
f"instl/{additional_info_map_file_name}")
additional_info_map_item = self.instlObj.info_map_table.get_file_item(
path_in_main_info_map)
if not additional_info_map_item: # zipped not found try the unzipped inf_map
additional_info_map_file_name = additional_info_map
path_in_main_info_map = config_vars.resolve_str(
f"instl/{additional_info_map}")
additional_info_map_item = self.instlObj.info_map_table.get_file_item(
path_in_main_info_map)
checksum = additional_info_map_item.checksum if additional_info_map_item else None
info_map_file_url = config_vars.resolve_str(
f"$(INSTL_FOLDER_BASE_URL)/{additional_info_map_file_name}"
)
local_copy_of_info_map_in = config_vars.resolve_str(
f"$(LOCAL_REPO_REV_BOOKKEEPING_DIR)/{additional_info_map}"
)
local_copy_of_info_map_out = utils.download_from_file_or_url(
in_url=info_map_file_url,
config_vars=config_vars,
in_target_path=local_copy_of_info_map_in,
translate_url_callback=connectionBase.translate_url,
cache_folder=self.instlObj.get_default_sync_dir(
"cache", make_dir=True),
expected_checksum=checksum)
self.instlObj.info_map_table.read_from_file(
local_copy_of_info_map_out)
self.instlObj.progress(
f"read info_map {info_map_file_url}")
new_have_info_map_path = os.fspath(
config_vars["NEW_HAVE_INFO_MAP_PATH"])
self.instlObj.info_map_table.write_to_file(
new_have_info_map_path,
field_to_write=('path', 'flags', 'revision', 'checksum',
'size'))
except Exception:
log.error(f"""Exception reading info_map: {info_map_file_url}""")
raise
def set_sync_locations_for_active_items(self):
# get_sync_folders_and_sources_for_active_iids returns: [(iid, direct_sync_indicator, source, source_tag, install_folder),...]
# direct_sync_indicator will be None unless the items has "direct_sync" section in index.yaml
# source is the relative path as it appears in index.yaml
# adjusted source is the source prefixed with $(SOURCE_PREFIX) -- it needed
# source_tag is one of '!dir', '!dir_cont', '!file'
# install_folder is where the sources should be copied to OR, in case of direct syn where they should be synced to
# install_folder will be None for those items that require only sync not copy (such as Icons)
#
# for each file item in the source this function will set the full path where to download the file: item.download_path
# and the top folder common to all items in a single source: item.download_root
sync_and_source = self.items_table.get_sync_folders_and_sources_for_active_iids(
)
items_to_update = list()
local_repo_sync_dir = os.fspath(config_vars["LOCAL_REPO_SYNC_DIR"])
config_vars.setdefault("ALL_SYNC_DIRS", local_repo_sync_dir)
for iid, direct_sync_indicator, source, source_tag, install_folder in sync_and_source:
direct_sync = self.get_direct_sync_status_from_indicator(
direct_sync_indicator)
resolved_source_parts = source.split("/")
if install_folder:
resolved_install_folder = config_vars.resolve_str(
install_folder)
else:
resolved_install_folder = install_folder
if source_tag in ('!dir', '!dir_cont'):
if direct_sync:
# for direct-sync source, if one of the sources is Info.xml and it exists on disk AND source & file
# have the same checksum, then no sync is needed at all. All the above is not relevant in repair mode.
need_to_sync = True
if not self.update_mode:
info_xml_item = self.info_map_table.get_file_item(
"/".join((source, "Info.xml")))
if info_xml_item:
info_xml_of_target = config_vars.resolve_str(
"/".join(
(resolved_install_folder,
resolved_source_parts[-1], "Info.xml")))
need_to_sync = not utils.check_file_checksum(
info_xml_of_target, info_xml_item.checksum)
if need_to_sync:
config_vars["ALL_SYNC_DIRS"].append(
resolved_install_folder)
item_paths = self.info_map_table.get_recursive_paths_in_dir(
dir_path=source, what="any")
self.progress(
f"mark for download {len(item_paths)} files of {iid}/{source}"
)
if source_tag == '!dir':
source_parent = "/".join(
resolved_source_parts[:-1])
for item in item_paths:
items_to_update.append({
"_id":
item['_id'],
"download_path":
config_vars.resolve_str("/".join(
(resolved_install_folder,
item['path'][len(source_parent) +
1:]))),
"download_root":
config_vars.resolve_str("/".join(
(resolved_install_folder,
resolved_source_parts[-1])))
})
else: # !dir_cont
source_parent = source
for item in item_paths:
items_to_update.append({
"_id":
item['_id'],
"download_path":
config_vars.resolve_str("/".join(
(resolved_install_folder,
item['path'][len(source_parent) +
1:]))),
"download_root":
resolved_install_folder
})
else:
num_ignored_files = self.info_map_table.ignore_file_paths_of_dir(
dir_path=source)
if num_ignored_files < 1:
num_ignored_files = "" # sqlite curs.rowcount does not always returns the number of effected rows
self.progress(
f"avoid download {num_ignored_files} files of {iid}, Info.xml has not changed"
)
else:
item_paths = self.info_map_table.get_recursive_paths_in_dir(
dir_path=source)
self.progress(
f"mark for download {len(item_paths)} files of {iid}/{source}"
)
for item in item_paths:
items_to_update.append({
"_id":
item['_id'],
"download_path":
config_vars.resolve_str("/".join(
(local_repo_sync_dir, item['path']))),
"download_root":
None
})
elif source_tag == '!file':
# if the file was wtarred and split it would have multiple items
items_for_file = self.info_map_table.get_required_paths_for_file(
source)
self.progress(
f"mark for download {len(items_for_file)} files of {iid}/{source}"
)
if direct_sync:
config_vars["ALL_SYNC_DIRS"].append(
resolved_install_folder)
for item in items_for_file:
items_to_update.append({
"_id":
item['_id'],
"download_path":
config_vars.resolve_str("/".join(
(resolved_install_folder, item['leaf']))),
"download_root":
config_vars.resolve_str(item.download_path)
})
else:
for item in items_for_file:
items_to_update.append(
{
"_id":
item['_id'],
"download_path":
config_vars.resolve_str("/".join(
(local_repo_sync_dir, item['path']))),
"download_root":
None
}
) # no need to set item.download_root here - it will not be used
self.info_map_table.update_downloads(items_to_update)
