def __init__(self, model_path: os.PathLike, leaf_template: str, inner_template: str, leaf_border: str, verbose: int) -> None:
"""
Initialize templates for the subroutines generation also the threshold for the leaf function size.
Additionally, instantiate basic internal resources (id generator and subroutines collector).
Use the DFS subroutines naming.
"""
assert os.path.isfile(model_path) and model_path.endswith(
'.java'), 'Fatal Error Splitter: invalid path to the DT java file.'
self.model_path = model_path
self.leaf_template = leaf_template
self.inner_template = inner_template
self.leaf_border = leaf_border
self.verbose = verbose
def get_id(verbose: int) -> int:
"""
Return unique id.
"""
id = 0
while True:
if verbose == 1:
print('Setting up subroutine {}..'.format(id))
yield id
id += 1
self.get_id = get_id(self.verbose)
self.subroutines = []
self.suffix = '\n'
def save_wav(y: array, sr: int, file: os.PathLike) -> None:
"""Save audio file to disk.
This function saves audio to disk using scipy.io.wavfile, with additional step
to convert input waveform to int16 unless it already is int16.
Parameters:
y(array): the audio data.
sr(int|None): the sample rate of the audio data. If sr does not match the actual audio data,
the resulting file will encounter play-back problems.
Notes:
The function only supports raw wav format.
"""
if y.ndim == 2 and y.shape[0] > y.shape[1]:
warnings.warn(
f'The audio array tried to saved has {y.shape[0]} channels ' +
f'and the wave length is {y.shape[1]}. It\'s that what you mean?' +
f'If not, try to tranpose the array before saving.')
if not file.endswith('.wav'):
raise ParameterError(
f'only .wav file supported, but dst file name is: {file}')
if sr <= 0:
raise ParameterError(
f'Sample rate should be larger than 0, recieved sr = {sr}')
if y.dtype not in ['int16', 'int8']:
warnings.warn(
f'input data type is {y.dtype}, will convert data to int16 format before saving'
)
y_out = depth_convert(y, 'int16')
else:
y_out = y
wavfile.write(file, sr, y_out.T)
def load_data(path: os.PathLike):
"""
Loading data from NumPy array format (.npy) or from MATLAB format (.mat)
:param path: Path to either .npy or .mat type file
:return: numpy array with loaded data
"""
if path.endswith(".npy"):
data = np.load(path)
elif path.endswith(".mat"):
mat = loadmat(path)
for key in mat.keys():
if "__" not in key:
data = mat[key]
break
else:
raise ValueError("This file type is not supported")
return data
def plot_pitch_class_histogram(x: np.array,
dest_path: os.PathLike,
by_instrument: bool = True):
"""Plot a histogram of pitches"""
if dest_path.endswith(".pgf"):
matplotlib.use("pgf")
xp = np.where(np.isin(x, [REST, SUSTAIN]), np.nan, x)
df = pd.DataFrame(xp, columns=TRACKS)
ax = sns.histplot(data=df)
ax.set_xlabel("MIDI pitch value")
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
plt.savefig(dest_path)
return ax
def plot_learning_curves(history_file: os.PathLike, dest_path: os.PathLike):
"""Plot the train vs. validation learning curves from a history file."""
if dest_path.endswith(".pgf"):
matplotlib.use("pgf")
df = pd.read_csv(history_file)
fig, ax = plt.subplots()
y = df.loss
y_val = df.val_loss
label = "Loss"
ax.plot(df.epoch, y, label=f"Training {label}")
ax.plot(df.epoch, y_val, label=f"Validation {label}")
ax.set_xlabel("Training Epochs")
ax.set_ylabel(f"{label} Score")
ax.set_ylim(0.0)
ax.legend(loc="best")
fig.tight_layout()
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
plt.savefig(dest_path, )
return (fig, ax)
def run_unix_split(fn: os.PathLike,
n_reads: int,
read_number: int,
output_prefix: os.PathLike = '',
gzip: bool = False,
compression_level: int = 5):
statement = []
if fn.endswith(".gz"):
cmd = f"""zcat {fn} | split -l {n_reads * 4} -d --additional-suffix=_{read_number}.fastq - {output_prefix}_part;"""
else:
cmd = f"cat {fn} | split -l {n_reads * 4} -d --additional-suffix=_{read_number}.fastq - {output_prefix}_part;"
statement.append(cmd)
if gzip:
statement.append(
f"ls {output_prefix}_part* | xargs -P 8 -n 1 gzip -{compression_level}"
)
subprocess.run(' '.join(statement), shell=True)
def _save_to_npz(file_path: os.PathLike, features: np.ndarray,
feature_name: str) -> None:
makedirs_safe(os.path.dirname(file_path))
if not file_path.endswith(".npz"):
file_path += ".npz"
file_path_backup = file_path + "_bak"
clean_backup_file = False
if os.path.isfile(file_path):
saved_features = dict(np.load(file_path))
os.rename(file_path, file_path_backup)
clean_backup_file = True
if feature_name in saved_features:
logging.info("Overriding {} in {}.".format(
feature_name, file_path))
saved_features[feature_name] = features
else:
saved_features = {feature_name: features}
try:
np.savez(file_path, **saved_features)
except:
if os.path.isfile(file_path_backup):
logging.error("Error when writing {}, restoring backup".format(
file_path))
if os.path.isfile(file_path):
os.remove(file_path)
os.rename(file_path_backup, file_path)
clean_backup_file = False
else:
logging.error("Error when writing {}.".format(file_path))
raise
if clean_backup_file:
os.remove(file_path_backup)
def locker(
file: os.PathLike,
password: bytes,
encrypting: typing.Optional[bool] = None,
remove: bool = True,
*,
ext: typing.Optional[str] = None,
newfile: typing.Optional[str] = None,
**kwargs,
) -> None:
"""Encrypts or decrypts files with AES algorithm.
See also:
:func:`lockerf` function for more details.
Args:
file (str):
The actual location of the file.
password (bytes, bytearray, memoryview):
Password to use to encrypt/decrypt the file. See :func:`lockerf`.
encrypting (bool):
Whether the file is being locked (encrypted) or not.
If ``encrypting`` is True, the file is encrypted no matter what
the extension is.
If ``encrypting`` is False, the file is decrypted no matter what
the extension is.
If ``encrypting`` is None (the default), it is guessed from the
file extension and the file header instead.
If encrypting is provided, argument ``ext`` is ignored.
remove (bool):
Whether to remove the file after encryption/decryption. Default
is True.
Keyword Arguments:
ext (str):
The extension to be used for the encrypted file. If None,
the default value ``.pyflk`` is used.
newfile (str):
The name of the file to be written upon. It must not be
already present. If None is provided (default), the name
of the ``file`` plus the extension is used.
**kwargs:
All other kwargs are passed to ``lockerf`` function.
Important:
The removal of file is **NOT** secure, because it uses
:py:func:`os.remove` to remove the file. With enough expertise and
time, the original file can be restored. If you want to remove the
original file securely, consider using ``shred`` or ``srm`` or some
other secure file deletion tools.
"""
if newfile and ext:
raise ValueError("newfile and ext are mutually exclusive")
ext = ext or EXTENSION
# guess encrypting if not provided
if encrypting is None:
encrypting = not file.endswith(ext)
# make newfile name if not provided
if newfile is None:
if encrypting:
newfile = file + ext
else:
newfile = os.path.splitext(file)[0]
try:
with open(file, "rb") as infile, open(newfile, "xb") as outfile:
lockerf(infile, outfile, password, encrypting, **kwargs)
except (TypeError, exc.DecryptionError):
# remove invalid file
os.remove(newfile)
raise
else:
# remove the original file
if remove:
os.remove(file)