def compute_musdb_metrics(musdb_root, subset, result_root, metrics_root,
instruments, metrics):
dataset = musdb.DB(root=musdb_root, is_wav=True, subsets=[subset])
museval.eval_mus_dir(dataset=dataset,
estimates_dir=result_root,
output_dir=metrics_root)
songs = glob(os.path.join(metrics_root, subset, '*.json'))
metrics = {
instrument: {k: []
for k in metrics}
for instrument in instruments
}
for song in songs:
with open(song, 'r') as stream:
data = json.load(stream)
for target in data['targets']:
instrument = target['name']
for metric in metrics:
sdr_med = np.median([
frame['metrics'][metric] for frame in target['frames']
if not np.isnan(frame['metrics'][metric])
])
metrics[instrument][metric].append(sdr_med)
return metrics
def evaluate(
adapter: str = AudioAdapterOption,
output_path: Path = AudioOutputOption,
stft_backend: STFTBackend = AudioSTFTBackendOption,
params_filename: str = ModelParametersOption,
mus_dir: Path = MUSDBDirectoryOption,
mwf: bool = MWFOption,
verbose: bool = VerboseOption,
) -> Dict:
"""
Evaluate a model on the musDB test dataset
"""
import numpy as np
configure_logger(verbose)
try:
import musdb
import museval
except ImportError:
logger.error("Extra dependencies musdb and museval not found")
logger.error("Please install musdb and museval first, abort")
raise Exit(10)
# Separate musdb sources.
songs = glob(join(mus_dir, EVALUATION_SPLIT, "*/"))
mixtures = [join(song, EVALUATION_MIXTURE) for song in songs]
audio_output_directory = join(output_path, EVALUATION_AUDIO_DIRECTORY)
separate(
deprecated_files=None,
files=mixtures,
adapter=adapter,
bitrate="128k",
codec=Codec.WAV,
duration=600.0,
offset=0,
output_path=join(audio_output_directory, EVALUATION_SPLIT),
stft_backend=stft_backend,
filename_format="{foldername}/{instrument}.{codec}",
params_filename=params_filename,
mwf=mwf,
verbose=verbose,
)
# Compute metrics with musdb.
metrics_output_directory = join(output_path, EVALUATION_METRICS_DIRECTORY)
logger.info("Starting musdb evaluation (this could be long) ...")
dataset = musdb.DB(root=mus_dir, is_wav=True, subsets=[EVALUATION_SPLIT])
museval.eval_mus_dir(
dataset=dataset,
estimates_dir=audio_output_directory,
output_dir=metrics_output_directory,
)
logger.info("musdb evaluation done")
# Compute and pretty print median metrics.
metrics = _compile_metrics(metrics_output_directory)
for instrument, metric in metrics.items():
logger.info(f"{instrument}:")
for metric, value in metric.items():
logger.info(f"{metric}: {np.median(value):.3f}")
return metrics
def _compute_musdb_metrics(arguments, musdb_root_directory,
audio_output_directory):
""" Generates musdb metrics fro previsouly computed audio estimation.
:param arguments: Entrypoint arguments.
:param audio_output_directory: Directory to get audio estimation from.
:returns: Path of generated metrics directory.
"""
metrics_output_directory = join(arguments.output_path, _METRICS_DIRECTORY)
get_logger().info('Starting musdb evaluation (this could be long) ...')
dataset = musdb.DB(root=musdb_root_directory,
is_wav=True,
subsets=[_SPLIT])
museval.eval_mus_dir(dataset=dataset,
estimates_dir=audio_output_directory,
output_dir=metrics_output_directory)
get_logger().info('musdb evaluation done')
return metrics_output_directory
# track, estimates, output_dir='bss_evals')
#print(scores)
return estimates
if __name__ == "__main__":
args = docopt(__doc__)
#print("Command line args:\n", args)
checkpoint_path = args["<checkpoint-path>"]
musdb_dir = args["<musdb-root>"]
device = torch.device("cuda" if use_cuda else "cpu")
print("using device:{}".format(device))
# build model
model = build_model().to(device)
# load checkpoint
model = load_checkpoint(checkpoint_path, model)
print("loading model from checkpoint:{}".format(checkpoint_path))
mus = musdb.DB(root_dir=musdb_dir, is_wav=True)
mus.run(evaluate, subsets="test", estimates_dir="estimates")
museval.eval_mus_dir(dataset=mus,
estimates_dir="estimates",
output_dir="bss_evals",
subsets="test",
parallel=True,
is_wav=True)
import musdb
import museval
from settings import *
mus = musdb.DB(dataset_path)
museval.eval_mus_dir(dataset=mus,
estimates_dir="sisec_estimates",
output_dir="musdb_out",
subsets="Test",
parallel=True)
import musdb
import museval
# initiate musdb
mus = musdb.DB()
# evaluate an existing estimate folder with wav files
museval.eval_mus_dir(
dataset=mus, # instance of musdb
estimates_dir='eval_test/MWF', # path to estimate folder
output_dir='./EST2', # set a folder to write eval json files
subsets="Test",
)
def test_evaluate_mus_dir(mus):
museval.eval_mus_dir(
dataset=mus, # instance of musdb
estimates_dir='data/EST', # path to estimate folder
output_dir='data/EST_scores_mus_dir', # set a folder to write eval
)