def decompress(self, compressed_path, reconstructed_path, original_file_info=None):
tr = tarlite.TarliteReader(tarlite_path=compressed_path)
bands_per_image = min(original_file_info["component_count"],
self.max_dimension_size // original_file_info["height"])
total_decompression_time = 0
recovered_components = []
with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir:
tr.extract_all(output_dir_path=tmp_dir)
for stack_index, start_band_index in enumerate(
range(0, original_file_info["component_count"], bands_per_image)):
compressed_stack_path = os.path.join(tmp_dir, str(stack_index))
with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, suffix=".pgm") as stack_path:
decompression_results = super().decompress(
compressed_path=compressed_stack_path, reconstructed_path=stack_path.name)
total_decompression_time += decompression_results.decompression_time_seconds
assert os.path.isfile(stack_path.name)
stack_array = pgm.read_pgm(input_path=stack_path.name)
limits = tuple(range(original_file_info["height"],
stack_array.shape[1],
original_file_info["height"]))
recovered_components.extend(np.hsplit(stack_array, limits))
e_sum = 0
for c in recovered_components:
assert len(c.shape) == 2
e_sum += c.shape[0] * c.shape[1]
assert e_sum == original_file_info['width'] * original_file_info['height'] * original_file_info[
'component_count'], \
f"Wrong number of recovered pixels {e_sum}, " \
f"expected {original_file_info['width'] * original_file_info['height'] * original_file_info['component_count']}."
assert len(recovered_components) == original_file_info["component_count"]
assert recovered_components[0].shape == (original_file_info["width"], original_file_info["height"])
recovered_array = np.dstack(recovered_components)
assert recovered_array.shape == (
original_file_info['width'], original_file_info['height'], original_file_info['component_count'])
if original_file_info["signed"]:
recovered_array = recovered_array.astype(np.int64)
recovered_array -= 2 ** ((8 * original_file_info["bytes_per_sample"]) - 1)
isets.dump_array_bsq(array=recovered_array, file_or_path=reconstructed_path,
dtype=isets.iproperties_row_to_numpy_dtype(image_properties_row=original_file_info))
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path, reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def test_read_write(self):
with tempfile.NamedTemporaryFile() as tmp_tarlite_file:
input_paths = glob.glob(
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"*.py"))
tw = tarlite.TarliteWriter(initial_input_paths=input_paths)
tw.write(output_path=tmp_tarlite_file.name)
tr = tarlite.TarliteReader(tarlite_path=tmp_tarlite_file.name)
with tempfile.TemporaryDirectory() as tmp_extract_dir:
tr.extract_all(output_dir_path=tmp_extract_dir)
for input_path in input_paths:
check_path = os.path.join(tmp_extract_dir,
os.path.basename(input_path))
assert filecmp.cmp(input_path, check_path)
def decompress_one(self,
compressed_path,
reconstructed_path,
original_file_info=None):
total_decompression_time = 0
with tempfile.NamedTemporaryFile(
dir=options.base_tmp_dir,
prefix=f"reconstructed_{os.path.basename(reconstructed_path)}",
suffix=".raw") as bil_le_file:
offset = 0
if original_file_info["signed"]:
reader = tarlite.TarliteReader(compressed_path)
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_extract_dir:
reader.extract_all(tmp_extract_dir)
import subprocess
invocation = f"ls -lah {tmp_extract_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".
format(status, invocation, output))
with open(
glob.glob(os.path.join(tmp_extract_dir,
"*.txt"))[0]) as si_file:
offset = int(si_file.read())
assert offset >= 0
os.path.getsize(tmp_extract_dir)
inner_compressed_path = glob.glob(
os.path.join(tmp_extract_dir, "*.mcalic"))[0]
dr = self.decompress_short_names(
compressed_path=inner_compressed_path,
reconstructed_path=bil_le_file.name,
original_file_info=original_file_info)
total_decompression_time += dr.decompression_time_seconds
else:
dr = self.decompress_short_names(
compressed_path=compressed_path,
reconstructed_path=bil_le_file.name,
original_file_info=original_file_info)
total_decompression_time += dr.decompression_time_seconds
original_dtype = isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info)
img = np.fromfile(bil_le_file.name,
dtype=original_dtype.replace(">", "<").replace(
"i", "u")).reshape(
original_file_info["height"],
original_file_info["component_count"],
original_file_info["width"])
if original_file_info["signed"]:
if offset != 0:
img = (img.astype("i4") - offset).astype(original_dtype)
else:
img = img.astype(original_dtype)
img = img.swapaxes(0, 1)
if original_file_info[
"big_endian"] and not original_file_info["signed"]:
# Signed file are already converted back to big_endian if necessary in the previous call to astype()
img = img.astype(original_dtype)
img.tofile(reconstructed_path)
try:
# The decoder always produces this file
os.remove("seqRec")
except FileNotFoundError:
pass
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
if original_file_info[
"width"] <= self.max_dimension_size and original_file_info[
"height"] <= self.max_dimension_size:
return self.decompress_one(compressed_path=compressed_path,
reconstructed_path=reconstructed_path,
original_file_info=original_file_info)
else:
tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path)
img = np.zeros(
(original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
dtype=isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info))
total_decompression_time = 0
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
tl_reader.extract_all(output_dir_path=tmp_dir)
invocation = f"ls -lah {tmp_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format(
status, invocation, output))
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
assert os.path.exists(small_compressed_path)
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_file_info = copy.copy(original_file_info)
small_file_info["height"] = original_file_info["height"] // self.split_height_count \
if y < self.split_height_count - 1 \
else original_file_info["height"] - (self.split_height_count - 1) * (
original_file_info["height"] // self.split_height_count)
small_file_info["width"] = original_file_info["width"] // self.split_width_count \
if x < self.split_width_count - 1 \
else original_file_info["width"] - (self.split_width_count - 1) * (
original_file_info["width"] // self.split_width_count)
dr = self.decompress_one(
compressed_path=small_compressed_path,
reconstructed_path=small_path,
original_file_info=small_file_info)
total_decompression_time += dr.decompression_time_seconds
small_array = isets.load_array_bsq(
file_or_path=small_path,
image_properties_row=small_file_info)
img[x * (original_file_info["width"] //
self.split_width_count):(
((x + 1) * (original_file_info["width"] //
self.split_width_count)
) if x < self.split_width_count -
1 else original_file_info["width"]), y *
(original_file_info["height"] //
self.split_height_count):(
((y + 1) * (original_file_info["height"] //
self.split_height_count)
) if y < self.split_height_count -
1 else original_file_info["height"]
), :] = small_array
isets.dump_array_bsq(array=img,
file_or_path=reconstructed_path)
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results