def create_download_instructions(self): self.instlObj.batch_accum.set_current_section('sync') already_synced_num_files, already_synced_num_bytes = self.instlObj.info_map_table.get_not_to_download_num_files_and_size() to_sync_num_files, bytes_to_sync = self.instlObj.info_map_table.get_to_download_num_files_and_size() var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", to_sync_num_files) var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync) # notify user how many files and bytes to sync self.instlObj.progress("{} of {} files to sync".format(to_sync_num_files, to_sync_num_files+already_synced_num_files)) self.instlObj.progress("{} of {} bytes to sync".format(bytes_to_sync, bytes_to_sync+already_synced_num_bytes)) if already_synced_num_files > 0: self.instlObj.batch_accum += self.instlObj.platform_helper.progress("{} files already in cache".format(already_synced_num_files), math.ceil(already_synced_num_files/4)) if to_sync_num_files == 0: return to_sync_num_files file_list = self.instlObj.info_map_table.get_download_items_sync_info() if False: # need to rethink how to calc mount point sizes efficiently mount_points_to_size = total_sizes_by_mount_point(file_list) for m_p in sorted(mount_points_to_size): free_bytes = shutil.disk_usage(m_p).free print(mount_points_to_size[m_p], "bytes to sync to drive", "".join(("'", m_p, "'")), free_bytes-mount_points_to_size[m_p], "bytes will remain") self.create_sync_folders() self.create_sync_urls(file_list) self.create_curl_download_instructions() self.instlObj.create_sync_folder_manifest_command("after-sync", back_ground=True) self.create_check_checksum_instructions(to_sync_num_files) return to_sync_num_files
def read_const_defines(self, a_node, *args, **kwargs): """ Read a !define_const sub-doc. All variables will be made const. Reading of internal state identifiers is allowed. __include__ is not allowed. """ del args, kwargs if a_node.isMapping(): for identifier, contents in a_node.items(): if identifier in ("__include__", "__include_if_exist__"): raise ValueError("!define_const doc cannot except __include__ and __include_if_exist__") var_stack.add_const_config_variable(identifier, "from !define_const section", *[item.value for item in contents])
def init_default_vars(self, initial_vars): if initial_vars: var_description = "from initial_vars" for var, value in initial_vars.items(): if isinstance(value, str): var_stack.add_const_config_variable(var, var_description, value) else: var_stack.add_const_config_variable(var, var_description, *value) var_description = "from InstlInstanceBase.init_default_vars" # read defaults/main.yaml main_defaults_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "main.yaml") self.read_yaml_file(main_defaults_file_path) # read defaults/compile-info.yaml compile_info_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "compile-info.yaml") if os.path.isfile(compile_info_file_path): self.read_yaml_file(compile_info_file_path) if "__COMPILATION_TIME__" not in var_stack: if var_stack.ResolveVarToStr("__INSTL_COMPILED__") == "True": var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "unknown compilation time") else: var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "(not compiled)") self.read_user_config()
def write_batch_file(self): if "__MAIN_OUT_FILE__" not in var_stack and "__MAIN_INPUT_FILE__" in var_stack: var_stack.add_const_config_variable("__MAIN_OUT_FILE__", "from write_batch_file", "$(__MAIN_INPUT_FILE__)-$(__MAIN_COMMAND__).$(BATCH_EXT)") self.batch_accum.set_current_section('pre') self.batch_accum += self.platform_helper.get_install_instructions_prefix() self.batch_accum.set_current_section('post') var_stack.set_var("TOTAL_ITEMS_FOR_PROGRESS_REPORT").append( str(self.platform_helper.num_items_for_progress_report)) self.batch_accum += self.platform_helper.get_install_instructions_postfix() lines = self.batch_accum.finalize_list_of_lines() for line in lines: if type(line) != str: raise TypeError("Not a string", type(line), line) # replace unresolved var references to native OS var references, e.g. $(HOME) would be %HOME% on Windows and ${HOME} one Mac lines_after_var_replacement = [value_ref_re.sub(self.platform_helper.var_replacement_pattern, line) for line in lines] output_text = "\n".join(lines_after_var_replacement) out_file = var_stack.ResolveVarToStr("__MAIN_OUT_FILE__") out_file = os.path.abspath(out_file) d_path, f_name = os.path.split(out_file) os.makedirs(d_path, exist_ok=True) with utils.write_to_file_or_stdout(out_file) as fd: fd.write(output_text) fd.write('\n') if out_file != "stdout": self.out_file_realpath = os.path.realpath(out_file) # chmod to 0777 so that file created under sudo, can be re-written under regular user. # However regular user cannot chmod for file created under sudo, hence the try/except try: os.chmod(self.out_file_realpath, 0o777) except Exception: pass else: self.out_file_realpath = "stdout" msg = " ".join( (self.out_file_realpath, str(self.platform_helper.num_items_for_progress_report), "progress items")) print(msg)
def create_download_instructions(self): """ remove files in sync folder that do not appear in the info map table """ self.instlObj.batch_accum.set_current_section('sync') file_list, bytes_to_sync = self.instlObj.info_map_table.get_to_download_files_and_size() var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", len(file_list)) var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync) # notify user how many files and bytes to sync print(len(file_list), "files to sync") print(bytes_to_sync, "bytes to sync") if len(file_list) == 0: return self.create_sync_folders() self.create_sync_urls(file_list) self.create_curl_download_instructions() self.create_check_checksum_instructions(file_list)
def init_from_cmd_line_options(self, cmd_line_options_obj): """ turn command line options into variables """ const_attrib_to_var = { "input_file": ("__MAIN_INPUT_FILE__", None), "output_file": ("__MAIN_OUT_FILE__", None), "props_file": ("__PROPS_FILE__", None), "config_file": ("__CONFIG_FILE__", None), "sh1_checksum": ("__SHA1_CHECKSUM__", None), "rsa_signature": ("__RSA_SIGNATURE__", None), "start_progress": ("__START_DYNAMIC_PROGRESS__", "0"), "total_progress": ("__TOTAL_DYNAMIC_PROGRESS__", "0"), "just_with_number": ("__JUST_WITH_NUMBER__", "0"), "limit_command_to": ("__LIMIT_COMMAND_TO__", None), "shortcut_path": ("__SHORTCUT_PATH__", None), "target_path": ("__SHORTCUT_TARGET_PATH__", None), "credentials": ("__CREDENTIALS__", None), "base_url": ("__BASE_URL__", None), "file_sizes_file": ("__FILE_SIZES_FILE__", None), "output_format": ("__OUTPUT_FORMAT__", "$(OUTPUT_FORMAT)") } for attrib, var in const_attrib_to_var.items(): attrib_value = getattr(cmd_line_options_obj, attrib) if attrib_value: var_stack.add_const_config_variable(var[0], "from command line options", *attrib_value) elif var[1] is not None: # there's a default var_stack.add_const_config_variable(var[0], "from default", var[1]) non_const_attrib_to_var = { "target_repo_rev": "TARGET_REPO_REV", "base_repo_rev": "BASE_REPO_REV", } for attrib, var in non_const_attrib_to_var.items(): attrib_value = getattr(cmd_line_options_obj, attrib) if attrib_value: var_stack.set_var(var, "from command line options").append(attrib_value[0]) if cmd_line_options_obj.command: self.the_command = cmd_line_options_obj.command self.fixed_command = self.the_command.replace('-', '_') var_stack.set_var("__MAIN_COMMAND__", "from command line options").append(cmd_line_options_obj.command) if hasattr(cmd_line_options_obj, "subject") and cmd_line_options_obj.subject is not None: var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options", cmd_line_options_obj.subject) else: var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options", "") if cmd_line_options_obj.state_file: var_stack.add_const_config_variable("__MAIN_STATE_FILE__", "from command line options", cmd_line_options_obj.state_file) if cmd_line_options_obj.run: var_stack.add_const_config_variable("__RUN_BATCH__", "from command line options", "yes") if cmd_line_options_obj.no_wtar_artifacts: var_stack.add_const_config_variable("__NO_WTAR_ARTIFACTS__", "from command line options", "yes") if cmd_line_options_obj.all_revisions: var_stack.add_const_config_variable("__ALL_REVISIONS__", "from command line options", "yes") if cmd_line_options_obj.dock_item_path: var_stack.add_const_config_variable("__DOCK_ITEM_PATH__", "from command line options", *cmd_line_options_obj.dock_item_path) if cmd_line_options_obj.dock_item_label: var_stack.add_const_config_variable("__DOCK_ITEM_LABEL__", "from command line options", *cmd_line_options_obj.dock_item_label) if cmd_line_options_obj.remove_from_dock: var_stack.add_const_config_variable("__REMOVE_FROM_DOCK__", "from command line options", "yes") if cmd_line_options_obj.restart_the_dock: var_stack.add_const_config_variable("__RESTART_THE_DOCK__", "from command line options", "yes") if cmd_line_options_obj.fail_exit_code: var_stack.add_const_config_variable("__FAIL_EXIT_CODE__", "from command line options", *cmd_line_options_obj.fail_exit_code) if cmd_line_options_obj.set_run_as_admin: var_stack.add_const_config_variable("__RUN_AS_ADMIN__", "from command line options", "yes") if cmd_line_options_obj.define: individual_definitions = cmd_line_options_obj.define[0].split(",") for definition in individual_definitions: name, value = definition.split("=") var_stack.set_var(name, "from command line define option").append(value)