def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
ignore_files = list(config_vars.get("WTAR_IGNORE_FILES", []))
self.what_to_unwtar: Path = utils.ExpandAndResolvePath(
self.what_to_unwtar)
if self.what_to_unwtar.is_file():
if utils.is_first_wtar_file(self.what_to_unwtar):
if self.where_to_unwtar:
destination_folder: Path = utils.ExpandAndResolvePath(
self.where_to_unwtar)
else:
destination_folder = self.what_to_unwtar.parent
self.unwtar_a_file(self.what_to_unwtar,
destination_folder,
no_artifacts=self.no_artifacts,
ignore=ignore_files,
copy_owner=self.copy_owner)
elif self.what_to_unwtar.is_dir():
if self.where_to_unwtar:
destination_folder: Path = Path(self.where_to_unwtar,
self.what_to_unwtar.name)
else:
destination_folder = self.what_to_unwtar
self.doing = f"""unwtar folder '{self.what_to_unwtar}' to '{destination_folder}''"""
if not can_skip_unwtar(self.what_to_unwtar, destination_folder):
for root, dirs, files in os.walk(self.what_to_unwtar,
followlinks=False):
# a hack to prevent unwtarring of the sync folder. Copy command might copy something
# to the top level of the sync folder.
if "bookkeeping" in dirs:
dirs[:] = []
log.debug(
f"skipping {root} because bookkeeping folder was found"
)
continue
root_Path = Path(root)
tail_folder = root_Path.relative_to(self.what_to_unwtar)
for a_file in files:
a_file_path = root_Path.joinpath(a_file)
if utils.is_first_wtar_file(a_file_path):
where_to_unwtar_the_file = destination_folder.joinpath(
tail_folder)
self.unwtar_a_file(a_file_path,
where_to_unwtar_the_file,
no_artifacts=self.no_artifacts,
ignore=ignore_files,
copy_owner=self.copy_owner)
else:
log.debug(
f"unwtar {self.what_to_unwtar} to {self.where_to_unwtar} skipping unwtarring because both folders have the same Info.xml file"
)
else:
raise FileNotFoundError(self.what_to_unwtar)
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_what_to_zip = utils.ExpandAndResolvePath(self.what_to_wzip)
if self.where_to_put_wzip:
target_wzip_file = utils.ExpandAndResolvePath(
self.where_to_put_wzip)
else:
target_wzip_file = resolved_what_to_zip.parent
if not target_wzip_file: # os.path.split might return empty string
target_wzip_file = Path.cwd()
if not target_wzip_file.is_file():
# assuming it's a folder
with MakeDir(target_wzip_file.parent,
report_own_progress=False) as md:
md()
target_wzip_file = target_wzip_file.joinpath(
resolved_what_to_zip.name + ".wzip")
self.doing = f"""wziping '{resolved_what_to_zip}' to '{target_wzip_file}'"""
zlib_compression_level = int(
config_vars.get("ZLIB_COMPRESSION_LEVEL", "8"))
with open(target_wzip_file, "wb") as wfd, open(resolved_what_to_zip,
"rb") as rfd:
wfd.write(zlib.compress(rfd.read(), zlib_compression_level))
def __call__(self, *args, **kwargs) -> None:
resolved_what_to_unwzip = utils.ExpandAndResolvePath(
self.what_to_unwzip)
target_unwzip_file = utils.ExpandAndResolvePath(
self.where_to_put_unwzip)
what_to_work_on_dir, what_to_work_on_leaf = os.path.split(
resolved_what_to_unwzip)
if not target_unwzip_file:
target_unwzip_file = resolved_what_to_unwzip.parent
if not target_unwzip_file: # os.path.split might return empty string
target_unwzip_file = Path.cwd()
if not target_unwzip_file.is_file():
# assuming it's a folder
with MakeDir(target_unwzip_file.parent,
report_own_progress=False) as md:
md()
if resolved_what_to_unwzip.name.endswith(".wzip"):
what_to_work_on_leaf = resolved_what_to_unwzip.stem
target_unwzip_file = os.path.join(target_unwzip_file,
what_to_work_on_leaf)
self.doing = f"""unzipping '{resolved_what_to_unwzip}' to '{target_unwzip_file}''"""
with open(resolved_what_to_unwzip,
"rb") as rfd, open(target_unwzip_file, "wb") as wfd:
decompressed = zlib.decompress(rfd.read())
wfd.write(decompressed)
def create_create_folders_instructions(self, folder_list: List[str]) -> None:
with self.batch_accum.sub_accum(Stage("create folders")) as create_folders_section:
kwargs_defaults = {'remove_obstacles': True, 'chowner': False, 'recursive_chmod': False}
third_party_folders = [utils.ExpandAndResolvePath(config_vars.resolve_str(os.fspath(p))) for p in config_vars.get("THIRD_PARTY_FOLDERS", []).list()]
for target_folder_path in folder_list:
target_folder_path = utils.ExpandAndResolvePath(config_vars.resolve_str(os.fspath(target_folder_path)))
our_folder = next((False for p in third_party_folders if p == target_folder_path), True)
create_folders_section += MakeDir(target_folder_path, chowner=our_folder, recursive_chmod=False)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_source = utils.ExpandAndResolvePath(self.source_file)
resolved_target = utils.ExpandAndResolvePath(self.target_file)
self.doing = f"Append {resolved_source} to {resolved_target}"
with open(self.target_file, "a") as wfd:
with open(self.source_file, "r") as rfd:
wfd.write(rfd.read())
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
if sys.platform == 'darwin':
# os.chmod is not recursive so call the system's chmod
if self.recursive:
self.doing = f"""change mode (recursive) of '{self.path}' to '{self.mode}''"""
return super().__call__(args, kwargs)
else:
resolved_path = utils.ExpandAndResolvePath(self.path)
path_stats = resolved_path.stat()
current_mode, current_mode_oct = stat.S_IMODE(
path_stats[stat.ST_MODE]), oct(
stat.S_IMODE(path_stats[stat.ST_MODE]))
flags, op = self.parse_symbolic_mode_mac(self.mode, path_stats)
mode_to_set = flags
if op == '+':
mode_to_set |= current_mode
elif op == '-':
mode_to_set = current_mode & ~flags
if mode_to_set != current_mode:
self.doing = f"""change mode of '{resolved_path}' to '{self.mode}''"""
os.chmod(resolved_path, mode_to_set)
else:
self.doing = f"""skip change mode of '{resolved_path}' mode is already '{mode_to_set}''"""
elif sys.platform == 'win32':
if self.recursive:
self.doing = f"""change mode (recursive) of '{self.path}' to '{self.mode}''"""
self.shell = True
return super().__call__(args, kwargs)
else:
resolved_path = utils.ExpandAndResolvePath(self.path)
who, perms, operation = self.parse_symbolic_mode_win(self.mode)
self.doing = f"""change mode of '{resolved_path}' to '{who}, {perms}, {operation}''"""
# on windows uncheck the read-only flag
if 'w' in self.mode:
os.chmod(resolved_path, stat.S_IWRITE)
accounts = list()
for name in who:
user, domain, type = win32security.LookupAccountName(
"", name)
accounts.append(user)
sd = win32security.GetFileSecurity(
os.fspath(resolved_path),
win32security.DACL_SECURITY_INFORMATION)
dacl = sd.GetSecurityDescriptorDacl()
for account in accounts:
dacl.AddAccessAllowedAce(win32security.ACL_REVISION, perms,
account)
sd.SetSecurityDescriptorDacl(1, dacl, 0)
win32security.SetFileSecurity(
os.fspath(resolved_path),
win32security.DACL_SECURITY_INFORMATION, sd)
def get_run_args(self, run_args) -> None:
resolved_curl_path = os.fspath(
utils.ExpandAndResolvePath(self.curl_path))
resolved_trg_path = os.fspath(utils.ExpandAndResolvePath(self.trg))
run_args.extend([
resolved_curl_path, "--insecure", "--fail", "--raw", "--silent",
"--show-error", "--connect-timeout", self.connect_time_out,
"--max-time", self.max_time, "--retry", self.retires,
"--retry-delay", self.retry_delay, "-o", resolved_trg_path,
self.src
])
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
if self.config_files:
for config_file in self.config_files:
config_file = utils.ExpandAndResolvePath(config_file)
self.read_yaml_file(config_file)
self.python_file = utils.ExpandAndResolvePath(self.python_file)
with utils.utf8_open_for_read(self.python_file, 'r') as rfd:
py_text = rfd.read()
py_compiled = compile(py_text, os.fspath(self.python_file), mode='exec', flags=0, dont_inherit=False, optimize=2)
exec(py_compiled, globals())
def get_run_args(self, run_args) -> None:
resolved_reshacker_path = os.fspath(
utils.ExpandAndResolvePath(self.reshacker_path))
if not os.path.isfile(resolved_reshacker_path):
raise FileNotFoundError(resolved_reshacker_path)
resolved_rc_file_path = os.fspath(
utils.ExpandAndResolvePath(self.rc_file_path))
run_args.extend([
resolved_reshacker_path, "-open", self.rc_file_path, "-action",
"compile"
])
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_path = utils.ExpandAndResolvePath(self.path)
for attempt in range(2):
try:
self.doing = f"""removing folder '{resolved_path}'"""
shutil.rmtree(resolved_path,
onerror=self.who_locks_file_error_dict)
break
except FileNotFoundError:
break
except NotADirectoryError:
kwargs_for_rm_file = self.all_kwargs_dict()
kwargs_for_rm_file['report_own_progress'] = False
kwargs_for_rm_file['recursive'] = False
with RmFile(resolved_path,
**kwargs_for_rm_file) as file_remover:
file_remover()
break
except PermissionError:
if attempt == 0:
log.info(f"Fixing permission for removing {resolved_path}")
from pybatch import FixAllPermissions
with FixAllPermissions(resolved_path,
report_own_progress=False,
recursive=True) as allower:
allower()
else:
raise
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
symlink_file_to_convert = utils.ExpandAndResolvePath(self.symlink_file_to_convert)
symlink_target = symlink_file_to_convert.read_text()
self.doing = f"""convert symlink file '{symlink_file_to_convert}' to real symlink to target '{symlink_target}'"""
symlink = Path(symlink_file_to_convert.parent, symlink_file_to_convert.stem)
it_was = None
if symlink.is_symlink():
with RmFile(symlink, report_own_progress=False, resolve_path=False) as rf:
rf()
it_was = "symlink"
elif symlink.is_file():
with RmFile(symlink, report_own_progress=False, resolve_path=False) as rf:
rf()
it_was = "file"
elif symlink.is_dir():
with RmDir(symlink, report_own_progress=False) as rd:
rd()
it_was = "folder"
if symlink.exists():
raise IsADirectoryError(f"{it_was} '{symlink}' a was found and could not be removed")
symlink.symlink_to(symlink_target)
symlink_file_to_convert.unlink()
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
folder_to_clean = utils.ExpandAndResolvePath(self.path_to_folder)
for item in os.scandir(folder_to_clean):
if item.name not in self.exclude:
with RmFileOrDir(item.path, own_progress_count=0) as rfod:
rfod()
def __call__(self, *args, **kwargs) -> None:
return # disable all dock operations since they cause crashes on Catalina
PythonBatchCommandBase.__call__(self, *args, **kwargs)
dock_util_command = list()
if self.remove:
dock_util_command.append("--remove")
if self.label_for_item:
dock_util_command.append(self.label_for_item)
if not self.restart_the_doc:
dock_util_command.append("--no-restart")
else:
if not self.path_to_item:
if self.restart_the_doc:
dock_util_command.append("--restart")
else:
log.warning("mac-dock confusing options, both --path and --restart were not supplied")
else:
dock_util_command.append("--add")
resolved_path_to_item = os.fspath(utils.ExpandAndResolvePath(self.path_to_item))
dock_util_command.append(resolved_path_to_item)
if self.label_for_item:
dock_util_command.append("--label")
dock_util_command.append(self.label_for_item)
dock_util_command.append("--replacing")
dock_util_command.append(self.label_for_item)
if not self.restart_the_doc:
dock_util_command.append("--no-restart")
self.doing = dock_util_command
utils.dock_util(dock_util_command)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_path = utils.ExpandAndResolvePath(
self.path, resolve_path=self.resolve_path)
for attempt in range(2):
try:
self.doing = f"""removing file '{resolved_path}'"""
resolved_path.unlink()
break
except FileNotFoundError:
break
except PermissionError as pex:
if attempt == 0:
# calling unlink on a folder raises PermissionError
if resolved_path.is_dir():
kwargs_for_rm_dir = self.all_kwargs_dict()
kwargs_for_rm_dir['report_own_progress'] = False
kwargs_for_rm_dir['recursive'] = True
with RmDir(resolved_path,
**kwargs_for_rm_dir) as dir_remover:
dir_remover()
break
else:
log.info(
f"Fixing permission for removing {resolved_path}")
from pybatch import FixAllPermissions
with FixAllPermissions(
resolved_path,
report_own_progress=False) as allower:
allower()
else:
self.who_locks_file_error_dict(Path.unlink, resolved_path)
raise
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
self.old_path = os.getcwd()
resolved_new_path = utils.ExpandAndResolvePath(self.new_path)
self.doing = f"""changing current directory to '{resolved_new_path}'"""
os.chdir(resolved_new_path)
assert os.getcwd() == os.fspath(resolved_new_path)
def read_include_node(self, i_node, *args, **kwargs):
if i_node.isScalar():
kwargs['original-path-to-file'] = i_node.value
resolved_file_name = config_vars.resolve_str(i_node.value)
self.read_yaml_file(resolved_file_name, *args, **kwargs)
elif i_node.isSequence():
for sub_i_node in i_node:
self.read_include_node(sub_i_node, *args, **kwargs)
elif i_node.isMapping():
if "url" in i_node:
file_was_downloaded_and_read = False
kwargs['original-path-to-file'] = i_node["url"].value
resolved_file_url = config_vars.resolve_str(
i_node["url"].value)
expected_checksum = None
if "checksum" in i_node:
expected_checksum = config_vars.resolve_str(
i_node["checksum"].value)
try:
file_path = utils.download_from_file_or_url(
in_url=resolved_file_url,
config_vars=config_vars,
in_target_path=None,
translate_url_callback=connectionBase.translate_url,
cache_folder=self.get_aux_cache_dir(make_dir=True),
expected_checksum=expected_checksum)
self.read_yaml_file(file_path, *args, **kwargs)
file_was_downloaded_and_read = True
except (FileNotFoundError, urllib.error.URLError):
ignore = kwargs.get('ignore_if_not_exist', False)
if ignore:
self.progress(
f"'ignore_if_not_exist' specified, ignoring FileNotFoundError for {resolved_file_url}"
)
else:
raise
if "copy" in i_node and file_was_downloaded_and_read:
self.batch_accum.set_current_section('post')
for copy_destination in i_node["copy"]:
need_to_copy = True
destination_file_resolved_path = utils.ExpandAndResolvePath(
config_vars.resolve_str(copy_destination.value))
if destination_file_resolved_path.is_file(
) and expected_checksum is not None:
checksums_match = utils.check_file_checksum(
file_path=destination_file_resolved_path,
expected_checksum=expected_checksum)
need_to_copy = not checksums_match
if need_to_copy:
self.batch_accum += MakeDir(
destination_file_resolved_path.parent,
chowner=True)
self.batch_accum += CopyFileToFile(
file_path,
destination_file_resolved_path,
hard_links=False,
copy_owner=True)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
folder = utils.ExpandAndResolvePath(self.path_to_folder)
for pattern in self.patterns:
list_to_remove = folder.glob(pattern)
for item in list_to_remove:
with RmFileOrDir(item, own_progress_count=0) as rfod:
rfod()
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
path_to_target = utils.ExpandAndResolvePath(self.path_to_target)
path_to_symlink = Path(os.path.expandvars(self.path_to_symlink))
self.doing = f"""create symlink '{path_to_symlink}' to target '{path_to_target}'"""
with RmFile(path_to_symlink, report_own_progress=False, resolve_path=False) as rf:
rf()
path_to_symlink.symlink_to(path_to_target)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
self.old_path = Path.cwd()
self.resolved_new_path = utils.ExpandAndResolvePath(self.new_path)
self.doing = f"""changing current directory to '{self.resolved_new_path}'"""
os.chdir(self.resolved_new_path)
assert self.resolved_new_path.samefile(
Path.cwd()), f"failed to cd into '{self.resolved_new_path}'"
def get_run_args(self, run_args) -> None:
self.path = utils.ExpandAndResolvePath(self.path)
run_args.extend(["icacls",
os.fspath(self.path),
"/grant",
"*S-1-1-0:(OI)(CI)F",
"/q"])
if self.recursive:
run_args.append("/t")
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_path = utils.ExpandAndResolvePath(self.path)
if resolved_path.is_dir():
os.utime(resolved_path)
else:
resolved_path.parent.mkdir(parents=True, exist_ok=True)
with open(resolved_path, 'a') as tfd:
os.utime(resolved_path, None)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_path = utils.ExpandAndResolvePath(self.path)
if resolved_path.is_file():
self.doing = f"""removing file'{resolved_path}'"""
resolved_path.unlink()
elif resolved_path.is_dir():
self.doing = f"""removing folder'{resolved_path}'"""
shutil.rmtree(resolved_path)
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
path_to_target = utils.ExpandAndResolvePath(self.path_to_target)
path_to_symlink = Path(os.path.expandvars(self.path_to_symlink))
try:
path_to_symlink.unlink()
except:
pass
self.doing = f"""create symlink '{path_to_symlink}' to target '{path_to_target}'"""
path_to_symlink.symlink_to(path_to_target)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_path = utils.ExpandAndResolvePath(self.path)
if resolved_path.is_dir():
os.utime(resolved_path)
else:
with MakeDir(resolved_path.parent,
report_own_progress=False) as md:
md()
with open(resolved_path, 'a') as tfd:
os.utime(resolved_path, None)
def get_run_args(self, run_args) -> None:
resolved_reshacker_path = os.fspath(
utils.ExpandAndResolvePath(self.reshacker_path))
if not os.path.isfile(resolved_reshacker_path):
raise FileNotFoundError(resolved_reshacker_path)
resolved_trg_path = os.fspath(utils.ExpandAndResolvePath(self.trg))
if not os.path.isfile(resolved_trg_path):
raise FileNotFoundError(resolved_trg_path)
resolved_resource_source_file = os.fspath(
utils.ExpandAndResolvePath(self.resource_source_file))
if not os.path.isfile(resolved_resource_source_file):
raise FileNotFoundError(resolved_resource_source_file)
run_args.extend([
resolved_reshacker_path, "-open", resolved_trg_path, "-save",
resolved_trg_path, "-resource", resolved_resource_source_file,
"-action", "addoverwrite"
])
if self.resource_type and self.resource_name:
run_args.extend(
["-mask", f"""{self.resource_type},{self.resource_name},0"""])
def __call__(self, *args, **kwargs) -> None:
PythonBatchCommandBase.__call__(self, *args, **kwargs)
resolved_folder_to_convert = utils.ExpandAndResolvePath(self.folder_to_convert)
for root, dirs, files in os.walk(resolved_folder_to_convert, followlinks=False):
for item in files:
item_path = Path(root, item)
if item_path.suffix == ".symlink":
self.last_symlink_file = os.fspath(item_path)
self.doing = f"""resolve symlink file '{self.last_symlink_file}'"""
with SymlinkFileToSymlink(item_path, own_progress_count=0) as symlink_converter:
symlink_converter()
def get_run_args(self, run_args) -> None:
path = os.fspath(utils.ExpandAndResolvePath(self.path))
self.doing = f"""changing flags '{",".join(self.flags)}' of file '{path}"""
per_system_flags = list(
filter(
None,
[self.flags_dict[sys.platform][flag] for flag in self.flags]))
if sys.platform == 'darwin':
self._create_run_args_mac(per_system_flags, path, run_args)
elif sys.platform == 'win32':
self._create_run_args_win(per_system_flags, path, run_args)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
if self.pattern is None:
log.wanging(
f"skip RmGlob of '{self.path_to_folder}' because pattern is None"
)
else:
folder = utils.ExpandAndResolvePath(self.path_to_folder)
list_to_remove = folder.glob(self.pattern)
for item in list_to_remove:
with RmFileOrDir(item, own_progress_count=0) as rfod:
rfod()
def get_run_args(self, run_args) -> None:
run_args.append("chown")
run_args.append("-f")
if self.recursive:
run_args.append("-R")
user_and_group = ""
if self.user_id != -1:
user_and_group += f"{self.user_id}"
if self.group_id != -1:
user_and_group += f":{self.group_id}"
run_args.append(user_and_group)
the_path = os.fspath(utils.ExpandAndResolvePath(self.path))
run_args.append(the_path)
def __call__(self, *args, **kwargs):
PythonBatchCommandBase.__call__(self, *args, **kwargs)
if (self.user_id, self.group_id) != (-1, -1):
# os.chown is not recursive so call the system's chown
if self.recursive:
self.doing = f"""change owner (recursive) of '{self.path}' to '{self.user_id}:{self.group_id}'"""
return super().__call__(args, kwargs)
else:
resolved_path = utils.ExpandAndResolvePath(self.path)
self.doing = f"""change owner of '{resolved_path}' to '{self.user_id}:{self.group_id}'"""
os.chown(resolved_path,
uid=int(self.user_id),
gid=int(self.group_id))
