def create_download_instructions(self):
self.instlObj.batch_accum.set_current_section('sync')
already_synced_num_files, already_synced_num_bytes = self.instlObj.info_map_table.get_not_to_download_num_files_and_size()
to_sync_num_files, bytes_to_sync = self.instlObj.info_map_table.get_to_download_num_files_and_size()
var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", to_sync_num_files)
var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync)
# notify user how many files and bytes to sync
self.instlObj.progress("{} of {} files to sync".format(to_sync_num_files, to_sync_num_files+already_synced_num_files))
self.instlObj.progress("{} of {} bytes to sync".format(bytes_to_sync, bytes_to_sync+already_synced_num_bytes))
if already_synced_num_files > 0:
self.instlObj.batch_accum += self.instlObj.platform_helper.progress("{} files already in cache".format(already_synced_num_files), math.ceil(already_synced_num_files/4))
if to_sync_num_files == 0:
return to_sync_num_files
file_list = self.instlObj.info_map_table.get_download_items_sync_info()
if False: # need to rethink how to calc mount point sizes efficiently
mount_points_to_size = total_sizes_by_mount_point(file_list)
for m_p in sorted(mount_points_to_size):
free_bytes = shutil.disk_usage(m_p).free
print(mount_points_to_size[m_p], "bytes to sync to drive", "".join(("'", m_p, "'")), free_bytes-mount_points_to_size[m_p], "bytes will remain")
self.create_sync_folders()
self.create_sync_urls(file_list)
self.create_curl_download_instructions()
self.instlObj.create_sync_folder_manifest_command("after-sync", back_ground=True)
self.create_check_checksum_instructions(to_sync_num_files)
return to_sync_num_files
def read_const_defines(self, a_node, *args, **kwargs):
""" Read a !define_const sub-doc. All variables will be made const.
Reading of internal state identifiers is allowed.
__include__ is not allowed.
"""
del args, kwargs
if a_node.isMapping():
for identifier, contents in a_node.items():
if identifier in ("__include__", "__include_if_exist__"):
raise ValueError("!define_const doc cannot except __include__ and __include_if_exist__")
var_stack.add_const_config_variable(identifier, "from !define_const section",
*[item.value for item in contents])
def init_default_vars(self, initial_vars):
if initial_vars:
var_description = "from initial_vars"
for var, value in initial_vars.items():
if isinstance(value, str):
var_stack.add_const_config_variable(var, var_description, value)
else:
var_stack.add_const_config_variable(var, var_description, *value)
var_description = "from InstlInstanceBase.init_default_vars"
# read defaults/main.yaml
main_defaults_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "main.yaml")
self.read_yaml_file(main_defaults_file_path)
# read defaults/compile-info.yaml
compile_info_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "compile-info.yaml")
if os.path.isfile(compile_info_file_path):
self.read_yaml_file(compile_info_file_path)
if "__COMPILATION_TIME__" not in var_stack:
if var_stack.ResolveVarToStr("__INSTL_COMPILED__") == "True":
var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "unknown compilation time")
else:
var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "(not compiled)")
self.read_user_config()
def write_batch_file(self):
if "__MAIN_OUT_FILE__" not in var_stack and "__MAIN_INPUT_FILE__" in var_stack:
var_stack.add_const_config_variable("__MAIN_OUT_FILE__", "from write_batch_file",
"$(__MAIN_INPUT_FILE__)-$(__MAIN_COMMAND__).$(BATCH_EXT)")
self.batch_accum.set_current_section('pre')
self.batch_accum += self.platform_helper.get_install_instructions_prefix()
self.batch_accum.set_current_section('post')
var_stack.set_var("TOTAL_ITEMS_FOR_PROGRESS_REPORT").append(
str(self.platform_helper.num_items_for_progress_report))
self.batch_accum += self.platform_helper.get_install_instructions_postfix()
lines = self.batch_accum.finalize_list_of_lines()
for line in lines:
if type(line) != str:
raise TypeError("Not a string", type(line), line)
# replace unresolved var references to native OS var references, e.g. $(HOME) would be %HOME% on Windows and ${HOME} one Mac
lines_after_var_replacement = [value_ref_re.sub(self.platform_helper.var_replacement_pattern, line) for line in lines]
output_text = "\n".join(lines_after_var_replacement)
out_file = var_stack.ResolveVarToStr("__MAIN_OUT_FILE__")
out_file = os.path.abspath(out_file)
d_path, f_name = os.path.split(out_file)
os.makedirs(d_path, exist_ok=True)
with utils.write_to_file_or_stdout(out_file) as fd:
fd.write(output_text)
fd.write('\n')
if out_file != "stdout":
self.out_file_realpath = os.path.realpath(out_file)
# chmod to 0777 so that file created under sudo, can be re-written under regular user.
# However regular user cannot chmod for file created under sudo, hence the try/except
try:
os.chmod(self.out_file_realpath, 0o777)
except Exception:
pass
else:
self.out_file_realpath = "stdout"
msg = " ".join(
(self.out_file_realpath, str(self.platform_helper.num_items_for_progress_report), "progress items"))
print(msg)
def create_download_instructions(self):
""" remove files in sync folder that do not appear in the info map table
"""
self.instlObj.batch_accum.set_current_section('sync')
file_list, bytes_to_sync = self.instlObj.info_map_table.get_to_download_files_and_size()
var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", len(file_list))
var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync)
# notify user how many files and bytes to sync
print(len(file_list), "files to sync")
print(bytes_to_sync, "bytes to sync")
if len(file_list) == 0:
return
self.create_sync_folders()
self.create_sync_urls(file_list)
self.create_curl_download_instructions()
self.create_check_checksum_instructions(file_list)
def init_from_cmd_line_options(self, cmd_line_options_obj):
""" turn command line options into variables """
const_attrib_to_var = {
"input_file": ("__MAIN_INPUT_FILE__", None),
"output_file": ("__MAIN_OUT_FILE__", None),
"props_file": ("__PROPS_FILE__", None),
"config_file": ("__CONFIG_FILE__", None),
"sh1_checksum": ("__SHA1_CHECKSUM__", None),
"rsa_signature": ("__RSA_SIGNATURE__", None),
"start_progress": ("__START_DYNAMIC_PROGRESS__", "0"),
"total_progress": ("__TOTAL_DYNAMIC_PROGRESS__", "0"),
"just_with_number": ("__JUST_WITH_NUMBER__", "0"),
"limit_command_to": ("__LIMIT_COMMAND_TO__", None),
"shortcut_path": ("__SHORTCUT_PATH__", None),
"target_path": ("__SHORTCUT_TARGET_PATH__", None),
"credentials": ("__CREDENTIALS__", None),
"base_url": ("__BASE_URL__", None),
"file_sizes_file": ("__FILE_SIZES_FILE__", None),
"output_format": ("__OUTPUT_FORMAT__", "$(OUTPUT_FORMAT)")
}
for attrib, var in const_attrib_to_var.items():
attrib_value = getattr(cmd_line_options_obj, attrib)
if attrib_value:
var_stack.add_const_config_variable(var[0], "from command line options", *attrib_value)
elif var[1] is not None: # there's a default
var_stack.add_const_config_variable(var[0], "from default", var[1])
non_const_attrib_to_var = {
"target_repo_rev": "TARGET_REPO_REV",
"base_repo_rev": "BASE_REPO_REV",
}
for attrib, var in non_const_attrib_to_var.items():
attrib_value = getattr(cmd_line_options_obj, attrib)
if attrib_value:
var_stack.set_var(var, "from command line options").append(attrib_value[0])
if cmd_line_options_obj.command:
self.the_command = cmd_line_options_obj.command
self.fixed_command = self.the_command.replace('-', '_')
var_stack.set_var("__MAIN_COMMAND__", "from command line options").append(cmd_line_options_obj.command)
if hasattr(cmd_line_options_obj, "subject") and cmd_line_options_obj.subject is not None:
var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options",
cmd_line_options_obj.subject)
else:
var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options", "")
if cmd_line_options_obj.state_file:
var_stack.add_const_config_variable("__MAIN_STATE_FILE__", "from command line options",
cmd_line_options_obj.state_file)
if cmd_line_options_obj.run:
var_stack.add_const_config_variable("__RUN_BATCH__", "from command line options", "yes")
if cmd_line_options_obj.no_wtar_artifacts:
var_stack.add_const_config_variable("__NO_WTAR_ARTIFACTS__", "from command line options", "yes")
if cmd_line_options_obj.all_revisions:
var_stack.add_const_config_variable("__ALL_REVISIONS__", "from command line options", "yes")
if cmd_line_options_obj.dock_item_path:
var_stack.add_const_config_variable("__DOCK_ITEM_PATH__", "from command line options", *cmd_line_options_obj.dock_item_path)
if cmd_line_options_obj.dock_item_label:
var_stack.add_const_config_variable("__DOCK_ITEM_LABEL__", "from command line options", *cmd_line_options_obj.dock_item_label)
if cmd_line_options_obj.remove_from_dock:
var_stack.add_const_config_variable("__REMOVE_FROM_DOCK__", "from command line options", "yes")
if cmd_line_options_obj.restart_the_dock:
var_stack.add_const_config_variable("__RESTART_THE_DOCK__", "from command line options", "yes")
if cmd_line_options_obj.fail_exit_code:
var_stack.add_const_config_variable("__FAIL_EXIT_CODE__", "from command line options", *cmd_line_options_obj.fail_exit_code)
if cmd_line_options_obj.set_run_as_admin:
var_stack.add_const_config_variable("__RUN_AS_ADMIN__", "from command line options", "yes")
if cmd_line_options_obj.define:
individual_definitions = cmd_line_options_obj.define[0].split(",")
for definition in individual_definitions:
name, value = definition.split("=")
var_stack.set_var(name, "from command line define option").append(value)