def create_download_instructions(self):
        self.instlObj.batch_accum.set_current_section('sync')

        already_synced_num_files, already_synced_num_bytes = self.instlObj.info_map_table.get_not_to_download_num_files_and_size()
        to_sync_num_files, bytes_to_sync = self.instlObj.info_map_table.get_to_download_num_files_and_size()
        var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", to_sync_num_files)
        var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync)

        # notify user how many files and bytes to sync
        self.instlObj.progress("{} of {} files to sync".format(to_sync_num_files, to_sync_num_files+already_synced_num_files))
        self.instlObj.progress("{} of {} bytes to sync".format(bytes_to_sync, bytes_to_sync+already_synced_num_bytes))

        if already_synced_num_files > 0:
            self.instlObj.batch_accum += self.instlObj.platform_helper.progress("{} files already in cache".format(already_synced_num_files), math.ceil(already_synced_num_files/4))

        if to_sync_num_files == 0:
            return to_sync_num_files

        file_list = self.instlObj.info_map_table.get_download_items_sync_info()
        if False:   # need to rethink how to calc mount point sizes efficiently
            mount_points_to_size = total_sizes_by_mount_point(file_list)

            for m_p in sorted(mount_points_to_size):
                free_bytes = shutil.disk_usage(m_p).free
                print(mount_points_to_size[m_p], "bytes to sync to drive", "".join(("'", m_p, "'")), free_bytes-mount_points_to_size[m_p], "bytes will remain")

        self.create_sync_folders()

        self.create_sync_urls(file_list)

        self.create_curl_download_instructions()
        self.instlObj.create_sync_folder_manifest_command("after-sync", back_ground=True)
        self.create_check_checksum_instructions(to_sync_num_files)
        return to_sync_num_files
Example #2
0
 def read_const_defines(self, a_node, *args, **kwargs):
     """ Read a !define_const sub-doc. All variables will be made const.
         Reading of internal state identifiers is allowed.
         __include__ is not allowed.
     """
     del args, kwargs
     if a_node.isMapping():
         for identifier, contents in a_node.items():
             if identifier in ("__include__", "__include_if_exist__"):
                 raise ValueError("!define_const doc cannot except __include__ and __include_if_exist__")
             var_stack.add_const_config_variable(identifier, "from !define_const section",
                                                 *[item.value for item in contents])
Example #3
0
    def init_default_vars(self, initial_vars):
        if initial_vars:
            var_description = "from initial_vars"
            for var, value in initial_vars.items():
                if isinstance(value, str):
                    var_stack.add_const_config_variable(var, var_description, value)
                else:
                    var_stack.add_const_config_variable(var, var_description, *value)

        var_description = "from InstlInstanceBase.init_default_vars"

        # read defaults/main.yaml
        main_defaults_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "main.yaml")
        self.read_yaml_file(main_defaults_file_path)

        # read defaults/compile-info.yaml
        compile_info_file_path = os.path.join(var_stack.ResolveVarToStr("__INSTL_DATA_FOLDER__"), "defaults", "compile-info.yaml")
        if os.path.isfile(compile_info_file_path):
            self.read_yaml_file(compile_info_file_path)
        if "__COMPILATION_TIME__" not in var_stack:
            if var_stack.ResolveVarToStr("__INSTL_COMPILED__") == "True":
                var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "unknown compilation time")
            else:
                var_stack.add_const_config_variable("__COMPILATION_TIME__", var_description, "(not compiled)")

        self.read_user_config()
Example #4
0
    def write_batch_file(self):
        if "__MAIN_OUT_FILE__" not in var_stack and "__MAIN_INPUT_FILE__" in var_stack:
            var_stack.add_const_config_variable("__MAIN_OUT_FILE__", "from write_batch_file",
                                                "$(__MAIN_INPUT_FILE__)-$(__MAIN_COMMAND__).$(BATCH_EXT)")

        self.batch_accum.set_current_section('pre')
        self.batch_accum += self.platform_helper.get_install_instructions_prefix()
        self.batch_accum.set_current_section('post')
        var_stack.set_var("TOTAL_ITEMS_FOR_PROGRESS_REPORT").append(
            str(self.platform_helper.num_items_for_progress_report))
        self.batch_accum += self.platform_helper.get_install_instructions_postfix()
        lines = self.batch_accum.finalize_list_of_lines()
        for line in lines:
            if type(line) != str:
                raise TypeError("Not a string", type(line), line)

        # replace unresolved var references to native OS var references, e.g. $(HOME) would be %HOME% on Windows and ${HOME} one Mac
        lines_after_var_replacement = [value_ref_re.sub(self.platform_helper.var_replacement_pattern, line) for line in lines]
        output_text = "\n".join(lines_after_var_replacement)

        out_file = var_stack.ResolveVarToStr("__MAIN_OUT_FILE__")
        out_file = os.path.abspath(out_file)
        d_path, f_name = os.path.split(out_file)
        os.makedirs(d_path, exist_ok=True)
        with utils.write_to_file_or_stdout(out_file) as fd:
            fd.write(output_text)
            fd.write('\n')

        if out_file != "stdout":
            self.out_file_realpath = os.path.realpath(out_file)
            # chmod to 0777 so that file created under sudo, can be re-written under regular user.
            # However regular user cannot chmod for file created under sudo, hence the try/except
            try:
                os.chmod(self.out_file_realpath, 0o777)
            except Exception:
                pass
        else:
            self.out_file_realpath = "stdout"
        msg = " ".join(
            (self.out_file_realpath, str(self.platform_helper.num_items_for_progress_report), "progress items"))
        print(msg)
Example #5
0
    def create_download_instructions(self):
        """ remove files in sync folder that do not appear in the info map table
        """
        self.instlObj.batch_accum.set_current_section('sync')

        file_list, bytes_to_sync = self.instlObj.info_map_table.get_to_download_files_and_size()
        var_stack.add_const_config_variable("__NUM_FILES_TO_DOWNLOAD__", "create_download_instructions", len(file_list))
        var_stack.add_const_config_variable("__NUM_BYTES_TO_DOWNLOAD__", "create_download_instructions", bytes_to_sync)

        # notify user how many files and bytes to sync
        print(len(file_list), "files to sync")
        print(bytes_to_sync, "bytes to sync")

        if len(file_list) == 0:
             return

        self.create_sync_folders()

        self.create_sync_urls(file_list)

        self.create_curl_download_instructions()
        self.create_check_checksum_instructions(file_list)
Example #6
0
    def init_from_cmd_line_options(self, cmd_line_options_obj):
        """ turn command line options into variables """
        const_attrib_to_var = {
            "input_file": ("__MAIN_INPUT_FILE__", None),
            "output_file": ("__MAIN_OUT_FILE__", None),
            "props_file": ("__PROPS_FILE__", None),
            "config_file": ("__CONFIG_FILE__", None),
            "sh1_checksum": ("__SHA1_CHECKSUM__", None),
            "rsa_signature": ("__RSA_SIGNATURE__", None),
            "start_progress": ("__START_DYNAMIC_PROGRESS__", "0"),
            "total_progress": ("__TOTAL_DYNAMIC_PROGRESS__", "0"),
            "just_with_number": ("__JUST_WITH_NUMBER__", "0"),
            "limit_command_to": ("__LIMIT_COMMAND_TO__", None),
            "shortcut_path": ("__SHORTCUT_PATH__", None),
            "target_path": ("__SHORTCUT_TARGET_PATH__", None),
            "credentials": ("__CREDENTIALS__", None),
            "base_url": ("__BASE_URL__", None),
            "file_sizes_file": ("__FILE_SIZES_FILE__", None),
            "output_format": ("__OUTPUT_FORMAT__", "$(OUTPUT_FORMAT)")
        }

        for attrib, var in const_attrib_to_var.items():
            attrib_value = getattr(cmd_line_options_obj, attrib)
            if attrib_value:
                var_stack.add_const_config_variable(var[0], "from command line options", *attrib_value)
            elif var[1] is not None:  # there's a default
                var_stack.add_const_config_variable(var[0], "from default", var[1])

        non_const_attrib_to_var = {
            "target_repo_rev": "TARGET_REPO_REV",
            "base_repo_rev": "BASE_REPO_REV",
        }

        for attrib, var in non_const_attrib_to_var.items():
            attrib_value = getattr(cmd_line_options_obj, attrib)
            if attrib_value:
                var_stack.set_var(var, "from command line options").append(attrib_value[0])

        if cmd_line_options_obj.command:
            self.the_command = cmd_line_options_obj.command
            self.fixed_command = self.the_command.replace('-', '_')
            var_stack.set_var("__MAIN_COMMAND__", "from command line options").append(cmd_line_options_obj.command)

        if hasattr(cmd_line_options_obj, "subject") and cmd_line_options_obj.subject is not None:
            var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options",
                                                cmd_line_options_obj.subject)
        else:
            var_stack.add_const_config_variable("__HELP_SUBJECT__", "from command line options", "")

        if cmd_line_options_obj.state_file:
            var_stack.add_const_config_variable("__MAIN_STATE_FILE__", "from command line options",
                                                cmd_line_options_obj.state_file)

        if cmd_line_options_obj.run:
            var_stack.add_const_config_variable("__RUN_BATCH__", "from command line options", "yes")

        if cmd_line_options_obj.no_wtar_artifacts:
            var_stack.add_const_config_variable("__NO_WTAR_ARTIFACTS__", "from command line options", "yes")

        if cmd_line_options_obj.all_revisions:
            var_stack.add_const_config_variable("__ALL_REVISIONS__", "from command line options", "yes")

        if cmd_line_options_obj.dock_item_path:
            var_stack.add_const_config_variable("__DOCK_ITEM_PATH__", "from command line options", *cmd_line_options_obj.dock_item_path)
        if cmd_line_options_obj.dock_item_label:
            var_stack.add_const_config_variable("__DOCK_ITEM_LABEL__", "from command line options", *cmd_line_options_obj.dock_item_label)
        if cmd_line_options_obj.remove_from_dock:
            var_stack.add_const_config_variable("__REMOVE_FROM_DOCK__", "from command line options", "yes")
        if cmd_line_options_obj.restart_the_dock:
            var_stack.add_const_config_variable("__RESTART_THE_DOCK__", "from command line options", "yes")
        if cmd_line_options_obj.fail_exit_code:
            var_stack.add_const_config_variable("__FAIL_EXIT_CODE__", "from command line options", *cmd_line_options_obj.fail_exit_code)
        if cmd_line_options_obj.set_run_as_admin:
            var_stack.add_const_config_variable("__RUN_AS_ADMIN__", "from command line options", "yes")

        if cmd_line_options_obj.define:
            individual_definitions = cmd_line_options_obj.define[0].split(",")
            for definition in individual_definitions:
                name, value = definition.split("=")
                var_stack.set_var(name, "from command line define option").append(value)