def get_wtar_total_checksum(wtar_file_path):
tar_total_checksum = None
try:
if not os.path.isfile(wtar_file_path):
wtar_file_path += ".aa"
if os.path.isfile(wtar_file_path):
wtar_file_paths = utils.find_split_files(wtar_file_path)
with utils.MultiFileReader("br", wtar_file_paths) as fd:
with tarfile.open(fileobj=fd) as tar:
tar_total_checksum = tar.pax_headers.get("total_checksum")
except Exception as ex:
pass # return None if there was exception from any reason
return tar_total_checksum
def wtar_ls_func(root_file_or_folder_path, ls_format):
listing_lines = list()
error_lines = list()
try:
what_to_work_on = utils.find_split_files(root_file_or_folder_path)
with utils.MultiFileReader("br", what_to_work_on) as fd:
with tarfile.open(fileobj=fd) as tar:
pax_headers = tar.pax_headers
for item in tar:
listing_lines.append(wtar_item_ls_func(item, ls_format))
listing_lines.append({
'W':
pax_headers.get("total_checksum", "no-total-checksum")
})
except Exception as ex:
error_lines.append([root_file_or_folder_path, ex.strerror])
return listing_lines, error_lines
def unwtar_a_file(self,
wtar_file_path: Path,
destination_folder: Path,
no_artifacts=False,
ignore=None,
copy_owner=False):
try:
wtar_file_paths = utils.find_split_files(wtar_file_path)
log.debug(f"unwtar {wtar_file_path} to {destination_folder}")
if ignore is None:
ignore = ()
first_wtar_file_dir, first_wtar_file_name = os.path.split(
wtar_file_paths[0])
destination_leaf_name = utils.original_name_from_wtar_name(
first_wtar_file_name)
destination_path = Path(destination_folder, destination_leaf_name)
self.doing = f"""unwtar file '{wtar_file_path}' to '{destination_folder} ({"already exists" if destination_path.exists() else "not exists"})'"""
do_the_unwtarring = True
with utils.MultiFileReader("br", wtar_file_paths) as fd:
with tarfile.open(fileobj=fd) as tar:
tar_total_checksum = tar.pax_headers.get("total_checksum")
# log.debug(f"total checksum for tarfile(s) {wtar_file_paths} {tar_total_checksum}")
if tar_total_checksum:
if os.path.exists(destination_path):
with utils.ChangeDirIfExists(destination_folder):
disk_total_checksum = utils.get_recursive_checksums(
destination_leaf_name, ignore=ignore).get(
"total_checksum",
"disk_total_checksum_was_not_found")
# log.debug(f"total checksum for destination {destination_folder} {disk_total_checksum}")
if disk_total_checksum == tar_total_checksum:
do_the_unwtarring = False
log.debug(
f"{wtar_file_paths[0]} skipping unwtarring because item exists and is identical to archive"
)
if do_the_unwtarring:
if os.path.exists(destination_path):
try:
utils.safe_remove_file_system_object(
destination_path, ignore_errors=False)
except PermissionError as pe:
ChmodAndChown(
destination_path,
"a+rw",
int(config_vars.get("ACTING_UID", -1)),
int(config_vars.get("ACTING_GID", -1)),
recursive=True,
own_progress_count=0)()
log.debug(
f"failed to remove {destination_path}, retrying after ChmodAndChow"
)
utils.safe_remove_file_system_object(
destination_path, ignore_errors=True)
log.debug(
f"2nd safe_remove_file_system_object on on {destination_path} done"
)
tar.extractall(destination_folder)
if copy_owner:
from pybatch import Chown
first_wtar_file_st = os.stat(wtar_file_paths[0])
# log.debug(f"copy_owner: {destination_folder} {first_wtar_file_st[stat.ST_UID]}:{first_wtar_file_st[stat.ST_GID]}")
Chown(destination_folder,
first_wtar_file_st[stat.ST_UID],
first_wtar_file_st[stat.ST_GID],
recursive=True)()
if no_artifacts:
for wtar_file in wtar_file_paths:
os.remove(wtar_file)
except OSError as e:
log.warning(
f"Invalid stream on split file with {wtar_file_paths[0]}")
raise e
except tarfile.TarError:
log.warning(
f"tarfile error while opening file {os.path.abspath(wtar_file_paths[0])}"
)
raise
def unwtar_a_file(self,
wtar_file_path: Path,
destination_folder: Path,
no_artifacts=False,
ignore=None,
copy_owner=False):
if ignore is None:
ignore = ()
try:
self.wtar_file_paths = utils.find_split_files(wtar_file_path)
log.debug(f"unwtar {wtar_file_path} to {destination_folder}")
destination_leaf_name = utils.original_name_from_wtar_name(
self.wtar_file_paths[0].name)
destination_path = destination_folder.joinpath(
destination_leaf_name)
self.doing = f"""unwtar file '{wtar_file_path}' to '{destination_folder} ({"already exists" if destination_path.exists() else "not exists"})'"""
do_the_unwtarring = True
with utils.MultiFileReader("br", self.wtar_file_paths) as fd:
with tarfile.open(fileobj=fd) as tar:
tar_total_checksum = tar.pax_headers.get("total_checksum")
# log.debug(f"total checksum for tarfile(s) {self.wtar_file_paths} {tar_total_checksum}")
if tar_total_checksum:
try:
if destination_path.exists():
with utils.ChangeDirIfExists(
destination_folder):
disk_total_checksum = utils.get_recursive_checksums(
destination_leaf_name, ignore=ignore
).get("total_checksum",
"disk_total_checksum_was_not_found")
# log.debug(f"total checksum for destination {destination_folder} {disk_total_checksum}")
if disk_total_checksum == tar_total_checksum:
log.debug(
f"{self.wtar_file_paths[0]} skipping unwtarring because item(s) exist and are identical to archive"
)
do_the_unwtarring = False
except:
# if checking checksum failed for any reason -> do the unwtarring
pass
if do_the_unwtarring:
with RmDir(destination_path,
report_own_progress=False,
recursive=True) as dir_remover:
# RmDir will also remove a file and will not raise if destination_path does not exist
dir_remover()
tar.extractall(destination_folder)
if copy_owner:
from pybatch import Chown
first_wtar_file_st = self.wtar_file_paths[0].stat()
# log.debug(f"copy_owner: {destination_folder} {first_wtar_file_st[stat.ST_UID]}:{first_wtar_file_st[stat.ST_GID]}")
Chown(destination_folder,
first_wtar_file_st[stat.ST_UID],
first_wtar_file_st[stat.ST_GID],
recursive=True)()
else:
log.info(
f"skip uwtar of {destination_path} because it exists and matches wtar file checksum"
)
if no_artifacts:
for wtar_file in self.wtar_file_paths:
with RmFile(wtar_file,
report_own_progress=False) as wtar_remover:
wtar_remover()
except OSError as e:
log.warning(
f"Invalid stream on split file with {self.wtar_file_paths[0]}")
raise e
except tarfile.TarError:
log.warning(
f"tarfile error while unwtarring file {self.wtar_file_paths[0]}"
)
raise