def __init__(self,
corpus_it_gen,
name,
subdivision=2,
sequence_size=3,
velocity_quantization=8,
max_transposition=3,
transpose_to_sounding_pitch=True,
cache_dir=None,
compute_statistics_flag=None):
"""
:param corpus_it_gen: calling this function returns an iterator
over chorales (as music21 scores)
:param name:
:param metadatas: list[Metadata], the list of used metadatas
:param subdivision: number of sixteenth notes per beat
:param cache_dir: directory where tensor_dataset is stored
"""
super(ArrangementPianorollDataset, self).__init__(cache_dir=cache_dir)
self.name = name
self.corpus_it_gen = corpus_it_gen
self.subdivision = subdivision # We use only on beats notes so far
assert sequence_size % 2 == 1
self.sequence_size = sequence_size
self.velocity_quantization = velocity_quantization
self.max_transposition = max_transposition
self.transpose_to_sounding_pitch = transpose_to_sounding_pitch
config = get_config()
reference_tessitura_path = config["reference_tessitura_path"]
simplify_instrumentation_path = config["simplify_instrumentation_path"]
self.dump_folder = config["dump_folder"]
self.statistic_folder = self.dump_folder + '/arrangement_pianoroll/statistics'
if os.path.isdir(self.statistic_folder):
shutil.rmtree(self.statistic_folder)
os.makedirs(self.statistic_folder)
# Reference tessitura for each instrument
with open(reference_tessitura_path, 'r') as ff:
tessitura = json.load(ff)
self.reference_tessitura = {
k: (music21.note.Note(v[0]), music21.note.Note(v[1]))
for k, v in tessitura.items()
}
self.observed_tessitura = {}
# Maps parts name found in mxml files to standard names
with open(simplify_instrumentation_path, 'r') as ff:
self.simplify_instrumentation = json.load(ff)
# Instrumentation used for learning
self.instrumentation = get_instrumentation()
# Compute statistics slows down the construction of the dataset
self.compute_statistics_flag = compute_statistics_flag
return
def __init__(self,
score_iterator,
subdivision,
savefolder_name,
sounding_pitch_boolean=False):
config = get_config()
# Dump folder
self.dump_folder = config['dump_folder']
self.savefolder_name = f'{self.dump_folder}/{savefolder_name}/statistics'
if os.path.isdir(self.savefolder_name):
shutil.rmtree(self.savefolder_name)
os.makedirs(self.savefolder_name)
self.num_bins = 30
# Simplify instrumentation
simplify_instrumentation_path = config['simplify_instrumentation_path']
with open(simplify_instrumentation_path, 'r') as ff:
self.simplify_instrumentation = json.load(ff)
self.instrument_grouping = get_instrument_grouping()
# Histogram with range of notes per instrument
self.tessitura = dict()
# Histogram with number of simultaneous notes
self.simultaneous_notes = dict()
# Histogram with the number of simultaneous notes per instrument
self.stat_dict = dict()
# Other stuffs used for parsing
self.score_iterator = score_iterator
self.subdivision = subdivision
self.sounding_pitch_boolean = sounding_pitch_boolean
# Write out paths here
self.simultaneous_notes_details_path = f'{self.savefolder_name}/simultaneous_notes_details.txt'
open(self.simultaneous_notes_details_path, 'w').close()
if self.sounding_pitch_boolean:
self.tessitura_path = f'{self.savefolder_name}/tessitura_sounding'
else:
self.tessitura_path = f'{self.savefolder_name}/tessitura'
if os.path.isdir(self.tessitura_path):
shutil.rmtree(self.tessitura_path)
os.makedirs(self.tessitura_path)
return
def get_all_datasets():
config = get_config()
annex_dataset = OrchestraIteratorGenerator(
folder_path=f"{config['database_path']}/Orchestration/orchestral",
subsets=["kunstderfuge"],
process_file=True,
)
return {
'arrangement': {
'dataset_class_name':
ArrangementDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'liszt_classical_archives',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'arrangement_large': {
'dataset_class_name':
ArrangementDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'imslp',
'liszt_classical_archives',
'bouliane',
'hand_picked_Spotify',
# 'debug'
],
num_elements=None,
),
'corpus_it_gen_instru_range':
annex_dataset
},
'arrangement_test': {
'dataset_class_name':
ArrangementDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'debug',
],
num_elements=None,
)
},
'arrangement_small': {
'dataset_class_name':
ArrangementDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'small_liszt_beethov',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'arrangement_frame_test': {
'dataset_class_name':
ArrangementFrameDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=['debug'],
num_elements=None,
)
},
'arrangement_frame': {
'dataset_class_name':
ArrangementFrameDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'liszt_classical_archives',
],
num_elements=None,
)
},
'arrangement_voice': {
'dataset_class_name':
ArrangementVoiceDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'liszt_classical_archives',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'arrangement_voice_small': {
'dataset_class_name':
ArrangementVoiceDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'small_liszt_beethov',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'arrangement_midiPiano': {
'dataset_class_name':
ArrangementMidipianoDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'liszt_classical_archives',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'arrangement_midiPiano_small': {
'dataset_class_name':
ArrangementMidipianoDataset,
'corpus_it_gen':
ArrangementIteratorGenerator(
arrangement_path=
f"{config['database_path']}/Orchestration/arrangement",
subsets=[
'small_liszt_beethov',
],
num_elements=None,
),
'corpus_it_gen_instru_range':
None
},
'bach_chorales': {
'dataset_class_name': ChoraleDataset,
'corpus_it_gen': music21.corpus.chorales.Iterator
},
'bach_chorales_beats': {
'dataset_class_name': ChoraleBeatsDataset,
'corpus_it_gen': music21.corpus.chorales.Iterator
},
'bach_chorales_beats_test': {
'dataset_class_name': ChoraleBeatsDataset,
'corpus_it_gen': ShortChoraleIteratorGen()
},
'bach_chorales_test': {
'dataset_class_name': ChoraleDataset,
'corpus_it_gen': ShortChoraleIteratorGen()
},
'lsdb_test': {
'dataset_class_name': LsdbDataset,
'corpus_it_gen': LeadsheetIteratorGenerator(num_elements=10)
},
'lsdb': {
'dataset_class_name': LsdbDataset,
'corpus_it_gen': LeadsheetIteratorGenerator(num_elements=None)
},
# 'folk':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(3, 4), (4, 4)]
# )
# },
# 'folk_test':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=10,
# has_chords=False,
# time_sigs=[(3, 4), (4, 4)]
# )
# },
# 'folk_4by4_test':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=100,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_3by4_test':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=100,
# has_chords=False,
# time_sigs=[(3, 4)]
# )
# },
# 'folk_3by4':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(3, 4)]
# )
# },
# 'folk_4by4chords':
# {
# 'dataset_class_name': FolkDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=True,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4measures_test':
# {
# 'dataset_class_name': FolkMeasuresDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=100,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4measures_test2':
# {
# 'dataset_class_name': FolkMeasuresDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=1,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4measures':
# {
# 'dataset_class_name': FolkMeasuresDataset,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4measurestr_test':
# {
# 'dataset_class_name': FolkMeasuresDatasetTranspose,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=1000,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4measurestr':
# {
# 'dataset_class_name': FolkMeasuresDatasetTranspose,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4nbars_test':
# {
# 'dataset_class_name': FolkDatasetNBars,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=100,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'folk_4by4nbars':
# {
# 'dataset_class_name': FolkDatasetNBars,
# 'corpus_it_gen': FolkIteratorGenerator(
# num_elements=None,
# has_chords=False,
# time_sigs=[(4, 4)]
# )
# },
# 'mnist':
# {
# 'dataset_class_name': MNISTDataset,
# },
}
import glob
import os
import re
import shutil
import numpy as np
import torch
from DatasetManager.arrangement.arrangement_helper import OrchestraIteratorGenerator
from DatasetManager.dataset_manager import DatasetManager
# Ugliness to the max, but I don't want to add Transformer to the venv path
import sys
from DatasetManager.config import get_config
config = get_config()
sys.path.append(config["transformer_path"])
from Transformer.reduction.reduc_data_processor import ReductionDataProcessor
from Transformer.transformer import Transformer
class Reducter:
def __init__(self,
writing_dir,
corpus_it_gen,
subdivision_model=2,
subdivision_read=4,
sequence_size=3,
velocity_quantization=2,
temperature=1.2):
"""