def test_rtf():
for dtype in [np.float32, np.float64]:
test_duration = 0.2
for tick_power in [-8,-6,-4]:
flops = 0
a = np.random.randn(300,300).astype(dtype)
start = time.process_time()
while time.process_time() - start < test_duration:
flops += a.size
a = np.random.randn(*a.shape).astype(dtype)
print("Flops/sec for randn with dtype={} is {} ".format(
dtype, flops / (time.process_time() - start)))
start = time.process_time()
while time.process_time() - start < test_duration:
flops += a.size
b = lilcom.compress(a, tick_power=tick_power)
print("Flops/sec for compression with dtype={} and tick_power={} is {} ".format(
dtype, tick_power, flops / (time.process_time() - start)))
start = time.process_time()
while time.process_time() - start < test_duration:
flops += a.size
a2 = lilcom.decompress(b)
print("Flops/sec for decompression with tick_power={} is {}".format(
tick_power, flops / (time.process_time() - start)))
def test_int16():
for bits_per_sample in [4, 5, 8]:
for axis in [-1, 1, 0, -2]:
a = ((np.random.rand(100 + bits_per_sample + axis,
200 + 10 * bits_per_sample + axis) * 65535) -
32768).astype(np.int16)
for use_out in [False, True]:
out_shape = lilcom.get_compressed_shape(
a.shape, axis, bits_per_sample)
b = lilcom.compress(a,
axis=axis,
bits_per_sample=bits_per_sample,
out=(np.empty(out_shape, dtype=np.int8)
if use_out else None))
# decompressing as int16, float or double should give the same result except
# it would be scaled by 1/32768
for d in [np.int16, np.float32, np.float64]:
c = lilcom.decompress(
b,
dtype=(None if use_out else d),
out=(np.empty(a.shape, dtype=d) if use_out else None))
a2 = a.astype(
np.float32) * (1.0 / 32768.0 if d != np.int16 else 1.0)
c2 = c.astype(np.float32)
rel_error = (np.fabs(a2 - c2)).sum() / (np.fabs(a2)).sum()
print(
"Relative error in int16 compression (decompressing as {}, axis={}, num-bits={}, use_out={}) is {}"
.format(d, axis, bits_per_sample, use_out, rel_error))
def test_rtf():
for dtype in [np.int16, np.float32, np.float64]:
# view the following as 100 channels where each channel
# is one second's worth of 16kHz-sampled data.
audio_time = 1000.0
# seconds
a = np.random.randn(int(audio_time), 16000)
if dtype == np.int16:
a *= 32768
a = a.astype(dtype)
for bits_per_sample in [4, 8]:
for lpc_order in [0, 1, 2, 4, 8]:
for axis in [0, 1]:
start = time.process_time()
b = lilcom.compress(a,
axis=axis,
bits_per_sample=bits_per_sample,
lpc_order=lpc_order)
mid = time.process_time()
c = lilcom.decompress(b, dtype=dtype)
end = time.process_time()
# f is a factor that we'll multiply the times by. The
# factor of 100.0 is to make the output percentages.
f = 100.0 / audio_time
print(
"RTF for dtype={}, bits-per-sample={}, lpc_order={}, axis={}, "
"compress/decompress/total RTF is: {:.3f}%,{:.3f}%,{:.3f}%"
.format(dtype, bits_per_sample, lpc_order, axis,
(mid - start) * f, (end - mid) * f,
(end - start) * f))
def store_feature_array(
feats: np.ndarray,
output_dir: Pathlike,
compress: bool = True,
lilcom_tick_power: int = -5
) -> Path:
"""
Store ``feats`` array on disk, using ``lilcom`` compression by default.
:param feats: a numpy ndarray containing features.
:param output_dir: a path to the directory where the features will be stored.
:param compress: a bool, whether the saved features should be compressed with ``lilcom``.
:param lilcom_tick_power: precision of ``lilcom`` compression - greater negative values (e.g. -8).
might be appropriate for non-log space features.
:return: a path to the file containing the stored array.
"""
output_dir = Path(output_dir)
(output_dir / 'storage').mkdir(parents=True, exist_ok=True)
output_features_path = (output_dir / 'storage' / str(uuid4())).with_suffix('.llc' if compress else '.npy')
if compress:
serialized_feats = lilcom.compress(feats, tick_power=lilcom_tick_power)
with open(output_features_path, 'wb') as f:
f.write(serialized_feats)
else:
np.save(output_features_path, feats, allow_pickle=False)
return output_features_path
def lilcomReconstruct(audioArray, lpcOrder):
""" This function will reconstruct the given audio array in form of a
conescutive compression and decompression procedure.
Args:
audioArray: A numpy array as the audio signal
lcpOrder: Same as lcpOrder in the main lilcom functions
Returns:
an Audio array with same size to the array passed as input which
is a result of compresion and decompresion
"""
bitPerSample = 6 # Issue make it passed by the operator
# bitsPerSample Should be recieved from settings
audioArray = audioArray.astype(np.float32)
outputShape = list(audioArray.shape)
outputShape[0] += 4
outputShape = tuple(outputShape)
outputArray = np.ndarray(outputShape, np.int8)
reconstructedArray = np.ndarray(audioArray.shape, np.int16)
c = lilcom.compress(audioArray, lpc_order=lpcOrder,
bits_per_sample=bitPerSample, axis=0)
reconstructedArray = lilcom.decompress(c, dtype=audioArray.dtype)
return reconstructedArray
def write(self, key: str, value: np.ndarray) -> str:
# Introduce a sub-directory that starts with the first 3 characters of the key, that is typically
# an auto-generated hash. This allows to avoid filesystem performance problems related to storing
# too many files in a single directory.
subdir = self.storage_path_ / key[:3]
subdir.mkdir(exist_ok=True)
output_features_path = (subdir / key).with_suffix(".llc")
serialized_feats = lilcom.compress(value, tick_power=self.tick_power)
with open(output_features_path, "wb") as f:
f.write(serialized_feats)
# Include sub-directory in the key, e.g. "abc/abcdef.llc"
return "/".join(output_features_path.parts[-2:])
def write(self, key: str, value: np.ndarray) -> str:
# We are manually adding the slash to join the base URL and the key.
if key.startswith("/"):
key = key[1:]
# Add lilcom extension.
if not key.endswith(".llc"):
key = key + ".llc"
output_features_url = f"{self.base_url}/{key}"
serialized_feats = lilcom.compress(value, tick_power=self.tick_power)
with SmartOpen.open(output_features_url, "wb") as f:
f.write(serialized_feats)
return key
def lilcom_compress_chunked(data: np.ndarray,
tick_power: int = -5,
do_regression=True,
chunk_size: int = 100) -> List[bytes]:
num_frames, num_feats = data.shape
compressed = []
for begin in range(0, num_frames, chunk_size):
compressed.append(
lilcom.compress(
data[begin:begin + chunk_size],
tick_power=tick_power,
do_regression=do_regression,
))
return compressed
def test_int16_lpc_order():
a = ((np.random.rand(100, 200) * 65535) - 32768).astype(np.int16)
for lpc in range(0, 15):
b = lilcom.compress(a, axis=-1, lpc_order=lpc)
c = lilcom.decompress(b, dtype=np.int16)
a2 = a.astype(np.float32)
c2 = c.astype(np.float32)
rel_error = (np.fabs(a2 - c2)).sum() / (np.fabs(a2)).sum()
print("Relative error in int16 with lpc order={} is {}".format(
lpc, rel_error))
def write(self, key: str, value: np.ndarray) -> str:
import smart_open
# We are manually adding the slash to join the base URL and the key.
if key.startswith('/'):
key = key[1:]
# Add lilcom extension.
if not key.endswith('.llc'):
key = key + '.llc'
output_features_url = f'{self.base_url}/{key}'
serialized_feats = lilcom.compress(value, tick_power=self.tick_power)
with smart_open.open(output_features_url,
'wb',
transport_params=self.transport_params) as f:
f.write(serialized_feats)
return key
def test_double():
a = np.random.randn(100, 200).astype(np.float64)
b = lilcom.compress(a, axis=-1)
c = lilcom.decompress(b, dtype=np.float64)
rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum()
print(
"Relative error in double compression, decompressing as double, is: ",
rel_error)
c = lilcom.decompress(b, dtype=np.float32)
rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum()
print("Relative error in double compression, decompressing as float, is: ",
rel_error)
def _process_and_store_recording(
self,
recording: Recording,
segmentation: Optional[SupervisionSegment] = None,
compressed: bool = True,
lilcom_tick_power: int = -8,
) -> List[Features]:
results = []
for channel in recording.channel_ids:
output_features_path = (
self.output_dir / 'storage' /
str(uuid4())).with_suffix('.llc' if compressed else '.npy')
samples = torch.from_numpy(
recording.load_audio(channels=channel, root_dir=self.root_dir))
# TODO: use augmentation manifest here
feats = self.feature_extractor.extract(
samples=samples,
sampling_rate=recording.sampling_rate).numpy()
if compressed:
# TODO: use segmentation manifest here
serialized_feats = lilcom.compress(
feats, tick_power=lilcom_tick_power)
with open(output_features_path, 'wb') as f:
f.write(serialized_feats)
else:
np.save(output_features_path, feats, allow_pickle=False)
results.append(
Features(
recording_id=recording.id,
channel_id=channel,
# TODO: revise start and duration with segmentation manifest info
start=0.0,
# We simplify the relationship between num_frames and duration - we guarantee that
# the duration is always num_frames * frame_shift
duration=feats.shape[0] *
self.feature_extractor.spectrogram_config.frame_shift,
type=self.feature_extractor.type,
num_frames=feats.shape[0],
num_features=feats.shape[1],
storage_type='lilcom' if compressed else 'numpy',
storage_path=str(output_features_path)))
return results
def lilcom_compress_chunked(
data: np.ndarray,
tick_power: int = -5,
do_regression=True,
chunk_size: int = 100,
temporal_dim: int = 0,
) -> List[bytes]:
assert temporal_dim < data.ndim
num_frames = data.shape[temporal_dim]
compressed = []
for begin in range(0, num_frames, chunk_size):
compressed.append(
lilcom.compress(
data[begin:begin + chunk_size],
tick_power=tick_power,
do_regression=do_regression,
))
return compressed
def test_float():
for bits_per_sample in [4, 6, 8]:
for axis in [-1, 1, 0, -2]:
for use_out in [False, True]:
a = np.random.randn(100 + bits_per_sample + axis, 200 +
bits_per_sample + axis).astype(np.float32)
out_shape = lilcom.get_compressed_shape(
a.shape, axis, bits_per_sample)
b = lilcom.compress(a,
axis=axis,
bits_per_sample=bits_per_sample,
out=(np.empty(out_shape, dtype=np.int8)
if use_out else None))
c = lilcom.decompress(b,
dtype=(None if use_out else np.float32),
out=(np.empty(a.shape, dtype=np.float32)
if use_out else None))
rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum()
print(
"Relative error in float compression (axis={}, bits-per-sample={}) is {}"
.format(axis, bits_per_sample, rel_error))
def main():
args = parser.parse_args()
if not os.path.isdir(args.output_dir):
raise RuntimeError("Expected directory {} to exist.".format(
args.output_dir))
output_scp_file = open(args.output_scp, "w", encoding="utf-8")
samp_rate = None
tot_time_reading = 0.0
tot_time_compressing = 0.0
tot_time_writing = 0.0
n = 0
with open(args.input_scp, "r", encoding="utf-8") as scp_file:
for line in scp_file:
fields = line.split()
if len(fields) < 2:
raise RuntimeError(
"Expected line in {} to have at least two fields, got: {}".
format(args.input_scp, line))
key = fields[0]
# The rxfilename is the rest of the fields... it may be a command.
rxfilename = ' '.join(fields[1:])
filename = (args.output_dir + os.path.sep + key + ".npy")
print(key, filename, file=output_scp_file)
if os.path.isfile(filename):
print("File exists {}, skipping".format(filename))
continue
start = time.perf_counter()
wave_file = file_utils.open_or_fd(rxfilename, encoding=None)
(f, s) = w2n.read_wave_as_numpy(wave_file)
tot_time_reading += time.perf_counter() - start
print("frame-rate={}, shape={}, dtype={}".format(
f, s.shape, s.dtype))
if samp_rate is None:
samp_rate = f
print("Sampling rate is {}".format(samp_rate))
elif samp_rate != f:
print("There is a sampling rate mismatch: {} vs {}".format(
samp_rate, s))
start = time.perf_counter()
compressed = lilcom.compress(s, axis=0)
tot_time_compressing += time.perf_counter() - start
start = time.perf_counter()
np.save(filename, compressed)
tot_time_writing += time.perf_counter() - start
n = n + 1
if n % 10 == 0:
print(
"Tot time reading/compressing/writing is: {},{},{}".format(
tot_time_reading, tot_time_compressing,
tot_time_writing))
output_scp_file.flush()
output_scp_file.close()
#!/usr/bin/env python3
import lilcom
import numpy as np
for shape in [(40, 50), (3, 4, 5), (1, 5, 7), (8, 1, 10), (100, 2, 57)]:
a = np.random.randn(*shape)
for power in [-15, -8, -6]:
b = lilcom.compress(a, power)
a2 = lilcom.decompress(b)
print("len(b) = ", len(b), ", bytes per number = ", (len(b) / a.size))
diff = (a2 - a)
mx = diff.max()
mn = diff.min()
limit = (2**(power - 1)) + 5.0e-05 # add a small margin to account for
# floating point roundoff.
print("max,min diff = {}, {}, expected magnitude was {}".format(
mx, mn, limit))
assert mx <= limit and -mn <= limit
def write(self, key: str, value: np.ndarray) -> str:
serialized_feats = lilcom.compress(value, tick_power=self.tick_power)
self.hdf.create_dataset(key, data=np.void(serialized_feats))
return key
def write(self, key: str, value: np.ndarray) -> str:
output_features_path = (self.storage_path_ / key).with_suffix('.llc')
serialized_feats = lilcom.compress(value, tick_power=self.tick_power)
with open(output_features_path, 'wb') as f:
f.write(serialized_feats)
return output_features_path.name
def write(self, key: str, value: np.ndarray) -> bytes:
return lilcom.compress(value)
