def get_disk_free_space(in_path):
retVal = 0
if 'Win' in utils.get_current_os_names():
secsPerCluster, bytesPerSec, nFreeCluster, totCluster = win32file.GetDiskFreeSpace(
in_path)
retVal = secsPerCluster * bytesPerSec * nFreeCluster
elif 'Mac' in utils.get_current_os_names():
st = os.statvfs(in_path)
retVal = st.f_bavail * st.f_frsize
return retVal
def read_file_or_url(in_file_or_url, path_searcher=None, encoding='utf-8', save_to_path=None, checksum=None):
need_to_download = not utils.check_file_checksum(save_to_path, checksum)
if not need_to_download:
# if save_to_path contains the correct data just read it by recursively
# calling read_file_or_url
return read_file_or_url(save_to_path, encoding=encoding)
match = protocol_header_re.match(in_file_or_url)
if not match: # it's a local file
local_file_path = in_file_or_url
if path_searcher is not None:
local_file_path = path_searcher.find_file(local_file_path)
if local_file_path:
if 'Win' in utils.get_current_os_names():
local_file_path = os.path.abspath(local_file_path)
else:
local_file_path = os.path.realpath(local_file_path)
else:
raise FileNotFoundError("Could not locate local file", local_file_path)
if encoding is None:
fd = open(local_file_path, "rb")
else:
fd = open(local_file_path, "r", encoding=encoding)
buffer = fd.read()
else:
session = pyinstl.connectionBase.connection_factory().get_session(in_file_or_url)
response = session.get(in_file_or_url, timeout=(33.05, 180.05))
response.raise_for_status()
buffer = response.text
buffer = utils.unicodify(buffer) # make sure text is unicode
if save_to_path and in_file_or_url != save_to_path:
with open(save_to_path, "w") as wfd:
wfd.write(buffer)
return buffer
def __init__(self, in_file_or_url, translate_url_callback=None, path_searcher=None, encoding='utf-8', verify_ssl=False):
self.local_file_path = None
self.url = None
self.custom_headers = None
self.encoding = encoding
self.verify_ssl = verify_ssl
self.fd = None
self._actual_path = in_file_or_url
match = protocol_header_re.match(in_file_or_url)
if not match: # it's a local file
self.local_file_path = in_file_or_url
if path_searcher is not None:
self.local_file_path = path_searcher.find_file(self.local_file_path)
if self.local_file_path:
if 'Win' in utils.get_current_os_names():
self.local_file_path = os.path.abspath(self.local_file_path)
else:
self.local_file_path = os.path.realpath(self.local_file_path)
else:
raise FileNotFoundError("Could not locate local file", self.local_file_path)
self._actual_path = self.local_file_path
else:
self.url = in_file_or_url
if translate_url_callback is not None:
self.url, self.custom_headers = translate_url_callback(self.url)
self._actual_path = self.url
def fix_path(self, in_some_path_to_fix):
""" On Windows: to overcome cUrl inability to handle path with unicode chars, we try to calculate the windows
short path (DOS style 8.3 chars). The function that does that, win32api.GetShortPathName,
does not work for paths that do not yet exist so we need to also create the folder.
However if the creation requires admin permissions - it could fail -
in which case we revert to using the long path.
"""
fixed_path = PurePath(in_some_path_to_fix)
if 'Win' in utils.get_current_os_names():
# to overcome cUrl inability to handle path with unicode chars, we try to calculate the windows
# short path (DOS style 8.3 chars). The function that does that, win32api.GetShortPathName,
# does not work for paths that do not yet exist so we need to also create the folder.
# However if the creation requires admin permissions - it could fail -
# in which case we revert to using the long path.
import win32api
fixed_path_parent = str(fixed_path.parent)
fixed_path_name = str(fixed_path.name)
if fixed_path_parent not in self.short_win_paths_cache:
try:
os.makedirs(fixed_path_parent, exist_ok=True)
short_parent_path = win32api.GetShortPathName(fixed_path_parent)
self.short_win_paths_cache[fixed_path_parent] = short_parent_path
except Exception as e: # failed to mkdir or get the short path? never mind, just use the full path
self.short_win_paths_cache[fixed_path_parent] = fixed_path_parent
log.warning(f"""warning creating short path failed for {fixed_path}, {e}, using long path""")
short_file_path = os.path.join(self.short_win_paths_cache[fixed_path_parent], fixed_path_name)
fixed_path = short_file_path.replace("\\", "\\\\")
return fixed_path
def read_file_or_url(in_file_or_url,
config_vars,
path_searcher=None,
encoding='utf-8',
save_to_path=None,
checksum=None,
connection_obj=None):
need_to_download = not utils.check_file_checksum(save_to_path, checksum)
if not need_to_download:
# if save_to_path contains the correct data just read it by recursively
# calling read_file_or_url
return read_file_or_url(save_to_path, config_vars, encoding=encoding)
match = protocol_header_re.match(os.fspath(in_file_or_url))
actual_file_path = in_file_or_url
if not match: # it's a local file
if path_searcher is not None:
actual_file_path = path_searcher.find_file(actual_file_path)
if actual_file_path:
if 'Win' in utils.get_current_os_names():
actual_file_path = os.path.abspath(actual_file_path)
else:
actual_file_path = os.path.realpath(actual_file_path)
else:
raise FileNotFoundError(
f"Could not locate local file {in_file_or_url}")
if encoding is None:
read_mod = "rb"
else:
read_mod = "r"
with open(actual_file_path, "r", encoding=encoding) as rdf:
buffer = rdf.read()
else:
assert connection_obj, "no connection_obj given"
session = connection_obj.get_session(in_file_or_url)
response = session.get(in_file_or_url, timeout=(33.05, 180.05))
response.raise_for_status()
buffer = response.text
buffer = utils.unicodify(buffer) # make sure text is unicode
if save_to_path and in_file_or_url != save_to_path:
with open(save_to_path, "w") as wfd:
utils.chown_chmod_on_fd(wfd)
wfd.write(buffer)
return buffer, actual_file_path
def __init__(self,
in_file_or_url,
config_vars,
translate_url_callback=None,
path_searcher=None,
encoding='utf-8',
verify_ssl=False) -> None:
self.local_file_path = None
self.url = None
self.custom_headers = None
self.encoding = encoding
self.verify_ssl = verify_ssl
self.fd = None
self._actual_path = in_file_or_url
match = protocol_header_re.match(os.fspath(in_file_or_url))
if not match: # it's a local file
self.local_file_path = in_file_or_url
if path_searcher is not None:
self.local_file_path = path_searcher.find_file(
self.local_file_path)
if self.local_file_path:
if 'Win' in utils.get_current_os_names():
self.local_file_path = os.path.abspath(
self.local_file_path)
else:
self.local_file_path = os.path.realpath(
self.local_file_path)
else:
raise FileNotFoundError(
f"Could not locate local file {self.local_file_path}")
self._actual_path = self.local_file_path
else:
self.url = in_file_or_url
if translate_url_callback is not None:
self.url, self.custom_headers = translate_url_callback(
self.url, config_vars)
self._actual_path = self.url
import json
from configVar import config_vars
from pybatch import PythonBatchRuntime
from pyinstl.cmdOptions import CommandLineOptions, read_command_line_options
from pyinstl.instlException import InstlException
import utils
#utils.set_max_open_files(2048)
from utils.log_utils import config_logger
log = logging.getLogger()
log.setLevel(logging.DEBUG)
current_os_names = utils.get_current_os_names()
os_family_name = current_os_names[0]
os_second_name = current_os_names[0]
if len(current_os_names) > 1:
os_second_name = current_os_names[1]
@lru_cache(maxsize=None)
def get_path_to_instl_app():
"""
@return: returns the path to this
"""
application_path = None
if getattr(sys, 'frozen', False):
application_path = Path(sys.executable).resolve()
elif __file__:
def disk_item_listing(files_or_folder_to_list,
ls_format='*',
output_format='text'):
""" Create a manifest of one or more folders or files
Format is a sequence of characters each specifying what details to include.
Details are listed in the order they appear in, unless specified as non-positional.
Options are:
'C': sha1 checksum (files only)
'D': Mac: if 'P' or 'p' is given "/" is appended to the path if item is a directory, non-positional
Win: <DIR> if item is directory empty string otherwise
'd': list only directories not files, non-positional.
'E': Mac: if 'P' or 'p' is given extra character is appended to the path, '@' for link, '*' for executable, '=' for socket, '|' for FIFO
Win: Not applicable
'f': list only files not directories, non-positional.
'g': Mac: gid
Win: Not applicable
'G': Mac: Group name or gid if name not found
Win: domain+"\\"+group name
'I': inode number (Mac only)
'L': Mac: number of links
Win: Not applicable
'M': a remark beginning in '#' and containing the data & time when the listing was done and the path to the item listed
non-positional. The remark appears before each top level item.
If the item was not found a remark will be written any way
'P': full path to the item
'p': partial path to the item - relative to the top folder listed, or if top item is a file - the file name without the path
'R': Mac: item's permissions in the format (-|d|l)rwxrwxrwx
Win: Not applicable
'S': size in bytes
'T': modification time, format is "%Y/%m/%d-%H:%M:%S" as used by time.strftime
'u': Mac: uid
Win: Not applicable
'U': Mac: User name or uid if name not found
Win: domain+"\\"+user name
'W': for wtar files only, total checksum
'*' if ls_format contains only '*' it is and alias to the default and means:
Mac: MIRLUGSTCPE
Win: MTDSUGCP
Note: if both 'd' and 'f' are not in ls_format disk_item_listing will act as if both are in ls_format
so 'SCp' actually means 'SCpfd'
"""
os_names = utils.get_current_os_names()
folder_ls_func = None
item_ls_func = None
if "Mac" in os_names:
if ls_format == '*':
ls_format = 'MIRLUGSTCPE'
folder_ls_func = unix_folder_ls
item_ls_func = unix_item_ls
elif "Win" in os_names:
if ls_format == '*':
ls_format = 'MTDSUGCP'
folder_ls_func = win_folder_ls
item_ls_func = win_item_ls
if 'f' not in ls_format and 'd' not in ls_format:
ls_format += 'fd'
add_remarks = 'M' in ls_format
ls_format = ls_format.replace('M', '')
listing_items = list()
error_items = list()
opening_remarks = list()
if add_remarks:
opening_remarks.append(
f"""# {datetime.datetime.today().isoformat()} listing of {files_or_folder_to_list}"""
)
if utils.is_first_wtar_file(files_or_folder_to_list):
listing_items, error_items = wtar_ls_func(files_or_folder_to_list,
ls_format=ls_format)
elif files_or_folder_to_list.is_dir():
listing_items, error_items = folder_ls_func(
files_or_folder_to_list,
ls_format=ls_format,
root_folder=files_or_folder_to_list)
elif files_or_folder_to_list.is_file() and 'f' in ls_format:
root_folder, _ = os.path.split(files_or_folder_to_list)
listings, errors = item_ls_func(files_or_folder_to_list,
ls_format=ls_format,
root_folder=root_folder)
listing_items.append(listings)
error_items.append(errors)
else:
opening_remarks.append(
f"""# folder was not found {files_or_folder_to_list}""")
if error_items:
opening_remarks.append(
f"error listing {len(error_items)} of {len(listing_items)+len(error_items)} items"
)
total_list = list()
if output_format == 'text':
total_list.extend(opening_remarks)
total_list.extend("Error: " + ", ".join(error)
for error in error_items)
total_list.extend(
list_of_dicts_describing_disk_items_to_text_lines(
listing_items, ls_format))
total_list.append(
""
) # line break at the end so not to be joined with the next line when printing to Terminal
elif output_format == 'dicts':
total_list.extend("Error: " + ", ".join(error)
for error in error_items)
for item in listing_items:
total_list.append(translate_item_dict_to_be_keyed_by_path(item))
elif output_format == 'json':
total_list.extend(
{"Error": + ", ".join(error)
for error in error_items})
total_list.append({
os.fspath(files_or_folder_to_list):
translate_json_key_names(listing_items)
})
if output_format == 'text':
retVal = "\n".join(total_list)
elif output_format == 'dicts':
retVal = total_list
elif output_format == 'json':
output_json = json.dumps(total_list,
indent=1,
default=utils.extra_json_serializer)
retVal = output_json
return retVal
- action before item
post_copy_item:
- action after item
post_copy_to_folder:
- action when leaving folder
"""
from collections import OrderedDict, defaultdict
from contextlib import contextmanager
import aYaml
import utils
import configVar
from configVar import var_stack
current_os_names = utils.get_current_os_names()
os_family_name = current_os_names[0]
def read_index_from_yaml(all_items_node):
retVal = dict()
for IID in all_items_node:
if IID in retVal:
print(IID, "found more than once in index")
else:
# print(IID, "not in all_items_node")
item = InstallItem(IID)
item.read_from_yaml_by_idd(all_items_node)
retVal[IID] = item
return retVal
