def get_disk_free_space(in_path): retVal = 0 if 'Win' in utils.get_current_os_names(): secsPerCluster, bytesPerSec, nFreeCluster, totCluster = win32file.GetDiskFreeSpace( in_path) retVal = secsPerCluster * bytesPerSec * nFreeCluster elif 'Mac' in utils.get_current_os_names(): st = os.statvfs(in_path) retVal = st.f_bavail * st.f_frsize return retVal
def read_file_or_url(in_file_or_url, path_searcher=None, encoding='utf-8', save_to_path=None, checksum=None): need_to_download = not utils.check_file_checksum(save_to_path, checksum) if not need_to_download: # if save_to_path contains the correct data just read it by recursively # calling read_file_or_url return read_file_or_url(save_to_path, encoding=encoding) match = protocol_header_re.match(in_file_or_url) if not match: # it's a local file local_file_path = in_file_or_url if path_searcher is not None: local_file_path = path_searcher.find_file(local_file_path) if local_file_path: if 'Win' in utils.get_current_os_names(): local_file_path = os.path.abspath(local_file_path) else: local_file_path = os.path.realpath(local_file_path) else: raise FileNotFoundError("Could not locate local file", local_file_path) if encoding is None: fd = open(local_file_path, "rb") else: fd = open(local_file_path, "r", encoding=encoding) buffer = fd.read() else: session = pyinstl.connectionBase.connection_factory().get_session(in_file_or_url) response = session.get(in_file_or_url, timeout=(33.05, 180.05)) response.raise_for_status() buffer = response.text buffer = utils.unicodify(buffer) # make sure text is unicode if save_to_path and in_file_or_url != save_to_path: with open(save_to_path, "w") as wfd: wfd.write(buffer) return buffer
def __init__(self, in_file_or_url, translate_url_callback=None, path_searcher=None, encoding='utf-8', verify_ssl=False): self.local_file_path = None self.url = None self.custom_headers = None self.encoding = encoding self.verify_ssl = verify_ssl self.fd = None self._actual_path = in_file_or_url match = protocol_header_re.match(in_file_or_url) if not match: # it's a local file self.local_file_path = in_file_or_url if path_searcher is not None: self.local_file_path = path_searcher.find_file(self.local_file_path) if self.local_file_path: if 'Win' in utils.get_current_os_names(): self.local_file_path = os.path.abspath(self.local_file_path) else: self.local_file_path = os.path.realpath(self.local_file_path) else: raise FileNotFoundError("Could not locate local file", self.local_file_path) self._actual_path = self.local_file_path else: self.url = in_file_or_url if translate_url_callback is not None: self.url, self.custom_headers = translate_url_callback(self.url) self._actual_path = self.url
def fix_path(self, in_some_path_to_fix): """ On Windows: to overcome cUrl inability to handle path with unicode chars, we try to calculate the windows short path (DOS style 8.3 chars). The function that does that, win32api.GetShortPathName, does not work for paths that do not yet exist so we need to also create the folder. However if the creation requires admin permissions - it could fail - in which case we revert to using the long path. """ fixed_path = PurePath(in_some_path_to_fix) if 'Win' in utils.get_current_os_names(): # to overcome cUrl inability to handle path with unicode chars, we try to calculate the windows # short path (DOS style 8.3 chars). The function that does that, win32api.GetShortPathName, # does not work for paths that do not yet exist so we need to also create the folder. # However if the creation requires admin permissions - it could fail - # in which case we revert to using the long path. import win32api fixed_path_parent = str(fixed_path.parent) fixed_path_name = str(fixed_path.name) if fixed_path_parent not in self.short_win_paths_cache: try: os.makedirs(fixed_path_parent, exist_ok=True) short_parent_path = win32api.GetShortPathName(fixed_path_parent) self.short_win_paths_cache[fixed_path_parent] = short_parent_path except Exception as e: # failed to mkdir or get the short path? never mind, just use the full path self.short_win_paths_cache[fixed_path_parent] = fixed_path_parent log.warning(f"""warning creating short path failed for {fixed_path}, {e}, using long path""") short_file_path = os.path.join(self.short_win_paths_cache[fixed_path_parent], fixed_path_name) fixed_path = short_file_path.replace("\\", "\\\\") return fixed_path
def read_file_or_url(in_file_or_url, config_vars, path_searcher=None, encoding='utf-8', save_to_path=None, checksum=None, connection_obj=None): need_to_download = not utils.check_file_checksum(save_to_path, checksum) if not need_to_download: # if save_to_path contains the correct data just read it by recursively # calling read_file_or_url return read_file_or_url(save_to_path, config_vars, encoding=encoding) match = protocol_header_re.match(os.fspath(in_file_or_url)) actual_file_path = in_file_or_url if not match: # it's a local file if path_searcher is not None: actual_file_path = path_searcher.find_file(actual_file_path) if actual_file_path: if 'Win' in utils.get_current_os_names(): actual_file_path = os.path.abspath(actual_file_path) else: actual_file_path = os.path.realpath(actual_file_path) else: raise FileNotFoundError( f"Could not locate local file {in_file_or_url}") if encoding is None: read_mod = "rb" else: read_mod = "r" with open(actual_file_path, "r", encoding=encoding) as rdf: buffer = rdf.read() else: assert connection_obj, "no connection_obj given" session = connection_obj.get_session(in_file_or_url) response = session.get(in_file_or_url, timeout=(33.05, 180.05)) response.raise_for_status() buffer = response.text buffer = utils.unicodify(buffer) # make sure text is unicode if save_to_path and in_file_or_url != save_to_path: with open(save_to_path, "w") as wfd: utils.chown_chmod_on_fd(wfd) wfd.write(buffer) return buffer, actual_file_path
def __init__(self, in_file_or_url, config_vars, translate_url_callback=None, path_searcher=None, encoding='utf-8', verify_ssl=False) -> None: self.local_file_path = None self.url = None self.custom_headers = None self.encoding = encoding self.verify_ssl = verify_ssl self.fd = None self._actual_path = in_file_or_url match = protocol_header_re.match(os.fspath(in_file_or_url)) if not match: # it's a local file self.local_file_path = in_file_or_url if path_searcher is not None: self.local_file_path = path_searcher.find_file( self.local_file_path) if self.local_file_path: if 'Win' in utils.get_current_os_names(): self.local_file_path = os.path.abspath( self.local_file_path) else: self.local_file_path = os.path.realpath( self.local_file_path) else: raise FileNotFoundError( f"Could not locate local file {self.local_file_path}") self._actual_path = self.local_file_path else: self.url = in_file_or_url if translate_url_callback is not None: self.url, self.custom_headers = translate_url_callback( self.url, config_vars) self._actual_path = self.url
import json from configVar import config_vars from pybatch import PythonBatchRuntime from pyinstl.cmdOptions import CommandLineOptions, read_command_line_options from pyinstl.instlException import InstlException import utils #utils.set_max_open_files(2048) from utils.log_utils import config_logger log = logging.getLogger() log.setLevel(logging.DEBUG) current_os_names = utils.get_current_os_names() os_family_name = current_os_names[0] os_second_name = current_os_names[0] if len(current_os_names) > 1: os_second_name = current_os_names[1] @lru_cache(maxsize=None) def get_path_to_instl_app(): """ @return: returns the path to this """ application_path = None if getattr(sys, 'frozen', False): application_path = Path(sys.executable).resolve() elif __file__:
def disk_item_listing(files_or_folder_to_list, ls_format='*', output_format='text'): """ Create a manifest of one or more folders or files Format is a sequence of characters each specifying what details to include. Details are listed in the order they appear in, unless specified as non-positional. Options are: 'C': sha1 checksum (files only) 'D': Mac: if 'P' or 'p' is given "/" is appended to the path if item is a directory, non-positional Win: <DIR> if item is directory empty string otherwise 'd': list only directories not files, non-positional. 'E': Mac: if 'P' or 'p' is given extra character is appended to the path, '@' for link, '*' for executable, '=' for socket, '|' for FIFO Win: Not applicable 'f': list only files not directories, non-positional. 'g': Mac: gid Win: Not applicable 'G': Mac: Group name or gid if name not found Win: domain+"\\"+group name 'I': inode number (Mac only) 'L': Mac: number of links Win: Not applicable 'M': a remark beginning in '#' and containing the data & time when the listing was done and the path to the item listed non-positional. The remark appears before each top level item. If the item was not found a remark will be written any way 'P': full path to the item 'p': partial path to the item - relative to the top folder listed, or if top item is a file - the file name without the path 'R': Mac: item's permissions in the format (-|d|l)rwxrwxrwx Win: Not applicable 'S': size in bytes 'T': modification time, format is "%Y/%m/%d-%H:%M:%S" as used by time.strftime 'u': Mac: uid Win: Not applicable 'U': Mac: User name or uid if name not found Win: domain+"\\"+user name 'W': for wtar files only, total checksum '*' if ls_format contains only '*' it is and alias to the default and means: Mac: MIRLUGSTCPE Win: MTDSUGCP Note: if both 'd' and 'f' are not in ls_format disk_item_listing will act as if both are in ls_format so 'SCp' actually means 'SCpfd' """ os_names = utils.get_current_os_names() folder_ls_func = None item_ls_func = None if "Mac" in os_names: if ls_format == '*': ls_format = 'MIRLUGSTCPE' folder_ls_func = unix_folder_ls item_ls_func = unix_item_ls elif "Win" in os_names: if ls_format == '*': ls_format = 'MTDSUGCP' folder_ls_func = win_folder_ls item_ls_func = win_item_ls if 'f' not in ls_format and 'd' not in ls_format: ls_format += 'fd' add_remarks = 'M' in ls_format ls_format = ls_format.replace('M', '') listing_items = list() error_items = list() opening_remarks = list() if add_remarks: opening_remarks.append( f"""# {datetime.datetime.today().isoformat()} listing of {files_or_folder_to_list}""" ) if utils.is_first_wtar_file(files_or_folder_to_list): listing_items, error_items = wtar_ls_func(files_or_folder_to_list, ls_format=ls_format) elif files_or_folder_to_list.is_dir(): listing_items, error_items = folder_ls_func( files_or_folder_to_list, ls_format=ls_format, root_folder=files_or_folder_to_list) elif files_or_folder_to_list.is_file() and 'f' in ls_format: root_folder, _ = os.path.split(files_or_folder_to_list) listings, errors = item_ls_func(files_or_folder_to_list, ls_format=ls_format, root_folder=root_folder) listing_items.append(listings) error_items.append(errors) else: opening_remarks.append( f"""# folder was not found {files_or_folder_to_list}""") if error_items: opening_remarks.append( f"error listing {len(error_items)} of {len(listing_items)+len(error_items)} items" ) total_list = list() if output_format == 'text': total_list.extend(opening_remarks) total_list.extend("Error: " + ", ".join(error) for error in error_items) total_list.extend( list_of_dicts_describing_disk_items_to_text_lines( listing_items, ls_format)) total_list.append( "" ) # line break at the end so not to be joined with the next line when printing to Terminal elif output_format == 'dicts': total_list.extend("Error: " + ", ".join(error) for error in error_items) for item in listing_items: total_list.append(translate_item_dict_to_be_keyed_by_path(item)) elif output_format == 'json': total_list.extend( {"Error": + ", ".join(error) for error in error_items}) total_list.append({ os.fspath(files_or_folder_to_list): translate_json_key_names(listing_items) }) if output_format == 'text': retVal = "\n".join(total_list) elif output_format == 'dicts': retVal = total_list elif output_format == 'json': output_json = json.dumps(total_list, indent=1, default=utils.extra_json_serializer) retVal = output_json return retVal
- action before item post_copy_item: - action after item post_copy_to_folder: - action when leaving folder """ from collections import OrderedDict, defaultdict from contextlib import contextmanager import aYaml import utils import configVar from configVar import var_stack current_os_names = utils.get_current_os_names() os_family_name = current_os_names[0] def read_index_from_yaml(all_items_node): retVal = dict() for IID in all_items_node: if IID in retVal: print(IID, "found more than once in index") else: # print(IID, "not in all_items_node") item = InstallItem(IID) item.read_from_yaml_by_idd(all_items_node) retVal[IID] = item return retVal