def test_rtf(): for dtype in [np.float32, np.float64]: test_duration = 0.2 for tick_power in [-8,-6,-4]: flops = 0 a = np.random.randn(300,300).astype(dtype) start = time.process_time() while time.process_time() - start < test_duration: flops += a.size a = np.random.randn(*a.shape).astype(dtype) print("Flops/sec for randn with dtype={} is {} ".format( dtype, flops / (time.process_time() - start))) start = time.process_time() while time.process_time() - start < test_duration: flops += a.size b = lilcom.compress(a, tick_power=tick_power) print("Flops/sec for compression with dtype={} and tick_power={} is {} ".format( dtype, tick_power, flops / (time.process_time() - start))) start = time.process_time() while time.process_time() - start < test_duration: flops += a.size a2 = lilcom.decompress(b) print("Flops/sec for decompression with tick_power={} is {}".format( tick_power, flops / (time.process_time() - start)))
def test_int16(): for bits_per_sample in [4, 5, 8]: for axis in [-1, 1, 0, -2]: a = ((np.random.rand(100 + bits_per_sample + axis, 200 + 10 * bits_per_sample + axis) * 65535) - 32768).astype(np.int16) for use_out in [False, True]: out_shape = lilcom.get_compressed_shape( a.shape, axis, bits_per_sample) b = lilcom.compress(a, axis=axis, bits_per_sample=bits_per_sample, out=(np.empty(out_shape, dtype=np.int8) if use_out else None)) # decompressing as int16, float or double should give the same result except # it would be scaled by 1/32768 for d in [np.int16, np.float32, np.float64]: c = lilcom.decompress( b, dtype=(None if use_out else d), out=(np.empty(a.shape, dtype=d) if use_out else None)) a2 = a.astype( np.float32) * (1.0 / 32768.0 if d != np.int16 else 1.0) c2 = c.astype(np.float32) rel_error = (np.fabs(a2 - c2)).sum() / (np.fabs(a2)).sum() print( "Relative error in int16 compression (decompressing as {}, axis={}, num-bits={}, use_out={}) is {}" .format(d, axis, bits_per_sample, use_out, rel_error))
def test_rtf(): for dtype in [np.int16, np.float32, np.float64]: # view the following as 100 channels where each channel # is one second's worth of 16kHz-sampled data. audio_time = 1000.0 # seconds a = np.random.randn(int(audio_time), 16000) if dtype == np.int16: a *= 32768 a = a.astype(dtype) for bits_per_sample in [4, 8]: for lpc_order in [0, 1, 2, 4, 8]: for axis in [0, 1]: start = time.process_time() b = lilcom.compress(a, axis=axis, bits_per_sample=bits_per_sample, lpc_order=lpc_order) mid = time.process_time() c = lilcom.decompress(b, dtype=dtype) end = time.process_time() # f is a factor that we'll multiply the times by. The # factor of 100.0 is to make the output percentages. f = 100.0 / audio_time print( "RTF for dtype={}, bits-per-sample={}, lpc_order={}, axis={}, " "compress/decompress/total RTF is: {:.3f}%,{:.3f}%,{:.3f}%" .format(dtype, bits_per_sample, lpc_order, axis, (mid - start) * f, (end - mid) * f, (end - start) * f))
def store_feature_array( feats: np.ndarray, output_dir: Pathlike, compress: bool = True, lilcom_tick_power: int = -5 ) -> Path: """ Store ``feats`` array on disk, using ``lilcom`` compression by default. :param feats: a numpy ndarray containing features. :param output_dir: a path to the directory where the features will be stored. :param compress: a bool, whether the saved features should be compressed with ``lilcom``. :param lilcom_tick_power: precision of ``lilcom`` compression - greater negative values (e.g. -8). might be appropriate for non-log space features. :return: a path to the file containing the stored array. """ output_dir = Path(output_dir) (output_dir / 'storage').mkdir(parents=True, exist_ok=True) output_features_path = (output_dir / 'storage' / str(uuid4())).with_suffix('.llc' if compress else '.npy') if compress: serialized_feats = lilcom.compress(feats, tick_power=lilcom_tick_power) with open(output_features_path, 'wb') as f: f.write(serialized_feats) else: np.save(output_features_path, feats, allow_pickle=False) return output_features_path
def lilcomReconstruct(audioArray, lpcOrder): """ This function will reconstruct the given audio array in form of a conescutive compression and decompression procedure. Args: audioArray: A numpy array as the audio signal lcpOrder: Same as lcpOrder in the main lilcom functions Returns: an Audio array with same size to the array passed as input which is a result of compresion and decompresion """ bitPerSample = 6 # Issue make it passed by the operator # bitsPerSample Should be recieved from settings audioArray = audioArray.astype(np.float32) outputShape = list(audioArray.shape) outputShape[0] += 4 outputShape = tuple(outputShape) outputArray = np.ndarray(outputShape, np.int8) reconstructedArray = np.ndarray(audioArray.shape, np.int16) c = lilcom.compress(audioArray, lpc_order=lpcOrder, bits_per_sample=bitPerSample, axis=0) reconstructedArray = lilcom.decompress(c, dtype=audioArray.dtype) return reconstructedArray
def write(self, key: str, value: np.ndarray) -> str: # Introduce a sub-directory that starts with the first 3 characters of the key, that is typically # an auto-generated hash. This allows to avoid filesystem performance problems related to storing # too many files in a single directory. subdir = self.storage_path_ / key[:3] subdir.mkdir(exist_ok=True) output_features_path = (subdir / key).with_suffix(".llc") serialized_feats = lilcom.compress(value, tick_power=self.tick_power) with open(output_features_path, "wb") as f: f.write(serialized_feats) # Include sub-directory in the key, e.g. "abc/abcdef.llc" return "/".join(output_features_path.parts[-2:])
def write(self, key: str, value: np.ndarray) -> str: # We are manually adding the slash to join the base URL and the key. if key.startswith("/"): key = key[1:] # Add lilcom extension. if not key.endswith(".llc"): key = key + ".llc" output_features_url = f"{self.base_url}/{key}" serialized_feats = lilcom.compress(value, tick_power=self.tick_power) with SmartOpen.open(output_features_url, "wb") as f: f.write(serialized_feats) return key
def lilcom_compress_chunked(data: np.ndarray, tick_power: int = -5, do_regression=True, chunk_size: int = 100) -> List[bytes]: num_frames, num_feats = data.shape compressed = [] for begin in range(0, num_frames, chunk_size): compressed.append( lilcom.compress( data[begin:begin + chunk_size], tick_power=tick_power, do_regression=do_regression, )) return compressed
def test_int16_lpc_order(): a = ((np.random.rand(100, 200) * 65535) - 32768).astype(np.int16) for lpc in range(0, 15): b = lilcom.compress(a, axis=-1, lpc_order=lpc) c = lilcom.decompress(b, dtype=np.int16) a2 = a.astype(np.float32) c2 = c.astype(np.float32) rel_error = (np.fabs(a2 - c2)).sum() / (np.fabs(a2)).sum() print("Relative error in int16 with lpc order={} is {}".format( lpc, rel_error))
def write(self, key: str, value: np.ndarray) -> str: import smart_open # We are manually adding the slash to join the base URL and the key. if key.startswith('/'): key = key[1:] # Add lilcom extension. if not key.endswith('.llc'): key = key + '.llc' output_features_url = f'{self.base_url}/{key}' serialized_feats = lilcom.compress(value, tick_power=self.tick_power) with smart_open.open(output_features_url, 'wb', transport_params=self.transport_params) as f: f.write(serialized_feats) return key
def test_double(): a = np.random.randn(100, 200).astype(np.float64) b = lilcom.compress(a, axis=-1) c = lilcom.decompress(b, dtype=np.float64) rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum() print( "Relative error in double compression, decompressing as double, is: ", rel_error) c = lilcom.decompress(b, dtype=np.float32) rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum() print("Relative error in double compression, decompressing as float, is: ", rel_error)
def _process_and_store_recording( self, recording: Recording, segmentation: Optional[SupervisionSegment] = None, compressed: bool = True, lilcom_tick_power: int = -8, ) -> List[Features]: results = [] for channel in recording.channel_ids: output_features_path = ( self.output_dir / 'storage' / str(uuid4())).with_suffix('.llc' if compressed else '.npy') samples = torch.from_numpy( recording.load_audio(channels=channel, root_dir=self.root_dir)) # TODO: use augmentation manifest here feats = self.feature_extractor.extract( samples=samples, sampling_rate=recording.sampling_rate).numpy() if compressed: # TODO: use segmentation manifest here serialized_feats = lilcom.compress( feats, tick_power=lilcom_tick_power) with open(output_features_path, 'wb') as f: f.write(serialized_feats) else: np.save(output_features_path, feats, allow_pickle=False) results.append( Features( recording_id=recording.id, channel_id=channel, # TODO: revise start and duration with segmentation manifest info start=0.0, # We simplify the relationship between num_frames and duration - we guarantee that # the duration is always num_frames * frame_shift duration=feats.shape[0] * self.feature_extractor.spectrogram_config.frame_shift, type=self.feature_extractor.type, num_frames=feats.shape[0], num_features=feats.shape[1], storage_type='lilcom' if compressed else 'numpy', storage_path=str(output_features_path))) return results
def lilcom_compress_chunked( data: np.ndarray, tick_power: int = -5, do_regression=True, chunk_size: int = 100, temporal_dim: int = 0, ) -> List[bytes]: assert temporal_dim < data.ndim num_frames = data.shape[temporal_dim] compressed = [] for begin in range(0, num_frames, chunk_size): compressed.append( lilcom.compress( data[begin:begin + chunk_size], tick_power=tick_power, do_regression=do_regression, )) return compressed
def test_float(): for bits_per_sample in [4, 6, 8]: for axis in [-1, 1, 0, -2]: for use_out in [False, True]: a = np.random.randn(100 + bits_per_sample + axis, 200 + bits_per_sample + axis).astype(np.float32) out_shape = lilcom.get_compressed_shape( a.shape, axis, bits_per_sample) b = lilcom.compress(a, axis=axis, bits_per_sample=bits_per_sample, out=(np.empty(out_shape, dtype=np.int8) if use_out else None)) c = lilcom.decompress(b, dtype=(None if use_out else np.float32), out=(np.empty(a.shape, dtype=np.float32) if use_out else None)) rel_error = (np.fabs(a - c)).sum() / (np.fabs(a)).sum() print( "Relative error in float compression (axis={}, bits-per-sample={}) is {}" .format(axis, bits_per_sample, rel_error))
def main(): args = parser.parse_args() if not os.path.isdir(args.output_dir): raise RuntimeError("Expected directory {} to exist.".format( args.output_dir)) output_scp_file = open(args.output_scp, "w", encoding="utf-8") samp_rate = None tot_time_reading = 0.0 tot_time_compressing = 0.0 tot_time_writing = 0.0 n = 0 with open(args.input_scp, "r", encoding="utf-8") as scp_file: for line in scp_file: fields = line.split() if len(fields) < 2: raise RuntimeError( "Expected line in {} to have at least two fields, got: {}". format(args.input_scp, line)) key = fields[0] # The rxfilename is the rest of the fields... it may be a command. rxfilename = ' '.join(fields[1:]) filename = (args.output_dir + os.path.sep + key + ".npy") print(key, filename, file=output_scp_file) if os.path.isfile(filename): print("File exists {}, skipping".format(filename)) continue start = time.perf_counter() wave_file = file_utils.open_or_fd(rxfilename, encoding=None) (f, s) = w2n.read_wave_as_numpy(wave_file) tot_time_reading += time.perf_counter() - start print("frame-rate={}, shape={}, dtype={}".format( f, s.shape, s.dtype)) if samp_rate is None: samp_rate = f print("Sampling rate is {}".format(samp_rate)) elif samp_rate != f: print("There is a sampling rate mismatch: {} vs {}".format( samp_rate, s)) start = time.perf_counter() compressed = lilcom.compress(s, axis=0) tot_time_compressing += time.perf_counter() - start start = time.perf_counter() np.save(filename, compressed) tot_time_writing += time.perf_counter() - start n = n + 1 if n % 10 == 0: print( "Tot time reading/compressing/writing is: {},{},{}".format( tot_time_reading, tot_time_compressing, tot_time_writing)) output_scp_file.flush() output_scp_file.close()
#!/usr/bin/env python3 import lilcom import numpy as np for shape in [(40, 50), (3, 4, 5), (1, 5, 7), (8, 1, 10), (100, 2, 57)]: a = np.random.randn(*shape) for power in [-15, -8, -6]: b = lilcom.compress(a, power) a2 = lilcom.decompress(b) print("len(b) = ", len(b), ", bytes per number = ", (len(b) / a.size)) diff = (a2 - a) mx = diff.max() mn = diff.min() limit = (2**(power - 1)) + 5.0e-05 # add a small margin to account for # floating point roundoff. print("max,min diff = {}, {}, expected magnitude was {}".format( mx, mn, limit)) assert mx <= limit and -mn <= limit
def write(self, key: str, value: np.ndarray) -> str: serialized_feats = lilcom.compress(value, tick_power=self.tick_power) self.hdf.create_dataset(key, data=np.void(serialized_feats)) return key
def write(self, key: str, value: np.ndarray) -> str: output_features_path = (self.storage_path_ / key).with_suffix('.llc') serialized_feats = lilcom.compress(value, tick_power=self.tick_power) with open(output_features_path, 'wb') as f: f.write(serialized_feats) return output_features_path.name
def write(self, key: str, value: np.ndarray) -> bytes: return lilcom.compress(value)