def get_batch(leadsheets, with_sample=False):
"""
Get a batch
leadsheets should be a list of dataset lists of (chord, melody) tuples, or just a dataset list of tuples
returns: chords, melodies
"""
if not isinstance(leadsheets[0], list):
leadsheets = [leadsheets]
sample_datasets = [random.randrange(len(leadsheets)) for _ in range(BATCH_SIZE)]
sample_fns = [random.choice(leadsheets[i]) for i in sample_datasets]
loaded_samples = [leadsheet.parse_leadsheet(lsfn) for lsfn in sample_fns]
sample_lengths = [leadsheet.get_leadsheet_length(c,m) for c,m in loaded_samples]
starts = [(0 if l==SEGMENT_LEN else random.randrange(0,l-SEGMENT_LEN,SEGMENT_STEP)) for l in sample_lengths]
sliced = [leadsheet.slice_leadsheet(c,m,s,s+SEGMENT_LEN) for (c,m),s in zip(loaded_samples, starts)]
res = list(zip(*sliced))
sample_sources = ["{}: starting at {} = bar {}".format(fn, start, start/(constants.WHOLE//constants.RESOLUTION_SCALAR)) for fn,start in zip(sample_fns, starts)]
if with_sample:
return res, sample_sources
else:
return res
def validate_generate(model, validation_leadsheets, generated_dir):
for lsfn in validation_leadsheets:
ch,mel = leadsheet.parse_leadsheet(lsfn)
batch = ([ch],[mel]), [lsfn]
curdir = os.path.join(generated_dir, os.path.splitext(os.path.basename(lsfn))[0])
os.makedirs(curdir)
generate(model, None, os.path.join(curdir, "generated"), with_vis=True, batch=batch)
def main(file, split, output=None):
c,m = leadsheet.parse_leadsheet(file)
lslen = leadsheet.get_leadsheet_length(c,m)
divwidth = int(split) * (constants.WHOLE//constants.RESOLUTION_SCALAR)
slices = [leadsheet.slice_leadsheet(c,m,s,s+divwidth) for s in range(0,lslen,divwidth)]
if output is None:
output = file + "-split"
for i, (chords, melody) in enumerate(slices):
leadsheet.write_leadsheet(chords, melody, '{}_{}.ls'.format(output,i))
def filter_leadsheets(leadsheets):
new_leadsheets = []
for lsfn in leadsheets:
print("---- {} ----".format(lsfn))
c, m = leadsheet.parse_leadsheet(lsfn, verbose=True)
length = leadsheet.get_leadsheet_length(c, m)
if length < SEGMENT_LEN:
print("Leadsheet {} is too short! Skipping...".format(lsfn))
else:
new_leadsheets.append(lsfn)
print("Found {} leadsheets.".format(len(leadsheets)))
return new_leadsheets
def validate_generate(model, validation_leadsheets, generated_dir):
for lsfn in validation_leadsheets:
ch, mel = leadsheet.parse_leadsheet(lsfn)
batch = ([ch], [mel]), [lsfn]
curdir = os.path.join(generated_dir,
os.path.splitext(os.path.basename(lsfn))[0])
os.makedirs(curdir)
generate(model,
None,
os.path.join(curdir, "generated"),
with_vis=True,
batch=batch)
def main(file, split, output=None):
c, m = leadsheet.parse_leadsheet(file)
lslen = leadsheet.get_leadsheet_length(c, m)
divwidth = int(split) * (constants.WHOLE // constants.RESOLUTION_SCALAR)
slices = [
leadsheet.slice_leadsheet(c, m, s, s + divwidth)
for s in range(0, lslen, divwidth)
]
if output is None:
output = file + "-split"
for i, (chords, melody) in enumerate(slices):
leadsheet.write_leadsheet(chords, melody, '{}_{}.ls'.format(output, i))
def main(files, output=None, verbose=False):
melody = []
chords = []
for f in files:
if verbose:
print(f)
nc,nm = leadsheet.parse_leadsheet(f, verbose)
melody.extend(nm)
chords.extend(nc)
if output is None:
output = files[0] + "-cat.ls"
leadsheet.write_leadsheet(chords, melody, output)
def filter_leadsheets(leadsheets):
new_leadsheets=[]
for lsfn in leadsheets:
print("---- {} ----".format(lsfn))
c,m = leadsheet.parse_leadsheet(lsfn, verbose=True)
length = leadsheet.get_leadsheet_length(c,m)
if length < SEGMENT_LEN:
print("Leadsheet {} is too short! Skipping...".format(lsfn))
else:
new_leadsheets.append(lsfn)
print("Found {} leadsheets.".format(len(leadsheets)))
return new_leadsheets
def findMaxAndMin():
maxMidi = 0
minMidi = float('inf')
for i in range(FIRSTLEADSHEET, LASTLEADSHEET + 1):
path = os.path.join(DATADIR, "{:04}.ls".format(i))
chords, melody = leadsheet.parse_leadsheet(path)
for note in melody:
if note[0] > maxMidi:
maxMidi = note[0]
if note[0] < minMidi and note[0] is not None:
minMidi = note[0]
return maxMidi, minMidi
def main():
trainingData = []
for i in range(FIRSTLEADSHEET, LASTLEADSHEET + 1):
path = os.path.join(DATADIR, "{:04}.ls".format(i))
chords, melody = leadsheet.parse_leadsheet(path)
currentSequence = []
for j in range(int(len(chords) / 2)):
chord = chords[j]
chordSequence = leadsheet.rotate(chord[1], chord[0])
currentSequence.append(chordSequence)
currentSequencePoint = 0
for j in range(len(melody)):
if currentSequencePoint >= 96:
break
note = melody[j]
for k in range(note[1]):
if currentSequencePoint >= 96:
break
if k == 0:
attackAndSustain = [1, 0]
else:
attackAndSustain = [0, 1]
# currentSequence[currentSequencePoint] += midiToOneHot(note[0]) + attackAndSustain
# if note[0] == None:
# midi = 90
# else:
# midi = note[0]
currentSequence[currentSequencePoint] += midiToOneHot(
note[0]) + attackAndSustain
currentSequencePoint += 1
trainingData.append(currentSequence)
# splitTrainingData = []
# splitLabels = []
# for i in range(len(trainingData)):
# for j in range(0, 192-50):
# splitTrainingData.append(trainingData[i][j:j+50])
# splitLabels.append(trainingData[i][j+50])
npTrainingData = np.array(trainingData)
print(npTrainingData.shape)
print(npTrainingData[0])
# npSplitTrainingData = np.array(splitTrainingData)
# npSplitLabels = np.array(splitLabels)
np.savez_compressed(SAVEFILE, data=npTrainingData)
def get_batch(leadsheets, with_sample=False):
"""
Get a batch
leadsheets should be a list of dataset lists of (chord, melody) tuples, or just a dataset list of tuples
returns: chords, melodies
"""
if not isinstance(leadsheets[0], list):
leadsheets = [leadsheets]
sample_datasets = [
random.randrange(len(leadsheets)) for _ in range(BATCH_SIZE)
]
sample_fns = [random.choice(leadsheets[i]) for i in sample_datasets]
loaded_samples = [leadsheet.parse_leadsheet(lsfn) for lsfn in sample_fns]
sample_lengths = [
leadsheet.get_leadsheet_length(c, m) for c, m in loaded_samples
]
starts = [(0 if l == SEGMENT_LEN else random.randrange(
0, l - SEGMENT_LEN, SEGMENT_STEP)) for l in sample_lengths]
sliced = [
leadsheet.slice_leadsheet(c, m, s, s + SEGMENT_LEN)
for (c, m), s in zip(loaded_samples, starts)
]
res = list(zip(*sliced))
sample_sources = [
"{}: starting at {} = bar {}".format(
fn, start,
start / (constants.WHOLE // constants.RESOLUTION_SCALAR))
for fn, start in zip(sample_fns, starts)
]
if with_sample:
return res, sample_sources
else:
return res
def load_data(directory=TRAINING_DIR): chords = np.array([]) melodies = np.array([]) for file in listdir(directory): c, m = ls.parse_leadsheet(directory + "/" + file) for i in range(len(c)): c[i] = np.append(c[i][1], pitchduration_to_circleofthirds(c[i][0] + 60,1)[0]) # deal with the root of the chord if len(chords) == 0: chords = np.array([c]) else: chords = np.append(chords, [c], axis=0) temp_melodies = np.array([]) for note in m: if len(temp_melodies) == 0: temp_melodies = np.array(pitchduration_to_circleofthirds(note[0], note[1])) else: temp_melodies = np.append(temp_melodies, pitchduration_to_circleofthirds(note[0], note[1]), axis=0) if len(melodies) == 0: melodies = np.array([temp_melodies]) else: melodies = np.append(melodies, [temp_melodies], axis=0) return chords, melodies
def parseLeadsheets(ldir, verbose=False):
"""
Parse a directory of leadsheets into a list of chords and melodies.
Inputs:
tdir: Path to directory containing leadsheets, i.e. "/transcriptions".
"""
asserterror_count = 0
keyerror_count = 0
bigrest_count = 0
numParsed = 0
clist = []
mlist = []
poslist = []
ckeylist = []
namelist = []
dlist = []
splist = []
alist = []
mlist_noskip = []
poslist_noskip = []
tlist_skip = []
tlist_noskip = []
if not ldir.endswith("/"):
ldir += "/"
if PITCH_REP == PITCH_MIDI:
print("PITCH: MIDI")
elif PITCH_REP == PITCH_INTERVAL:
print("PITCH: INTERVAL")
elif PITCH_REP == PITCH_CHORD:
print("PITCH: CHORD")
##################### Get roadmaps to get chords & bricks
# Clear temp directories
folder = './temp_leadsheets/'
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
#elif os.path.isdir(file_path): shutil.rmtree(file_path)
except Exception as e:
print(e)
# Generate roadmaps in temp directories
max_length = 0 # For tracking purposes, usually 24 is good
lowest_note = 200 # For tracking purposes
highest_note = 0 # For tracking purposes
roadmaps = roadmap_parser.generateRoadmaps(ldir,
"./temp_leadsheets",
doTranspose=False)
##################### Grab chords and durations from roadmaps
# i (leadsheet index)
# 0 ident (leadsheet), 1 roadmap for leadsheet
# 0 ident (roadmap), 1 actual roadmap content
# 0 blocks list
# 0 ident (blocks), 1 actual blocks content
# Chord durations are 480 for 1 measure
rcdlist = []
totalbrickdur = 0
numChords = 0
numBricks = 0
possiblepitches = {}
for j in range(len(roadmaps)):
bricks = roadmaps[j][1][1][0][1]
chordsdurs = []
for brick in bricks:
chordsdurs_brick = []
chords = roadmap_parser.chordFinder(brick, durOn=True)
numChords += len(chords)
for i in range(len(chords)):
#try:
cval = leadsheet.parse_chord(chords[i][0])
# except KeyError:
# print("KeyError at Roadmap ",j)
# print(os.listdir(ldir)[j])
# return
dur = int(int(chords[i][1]) / constants.RESOLUTION_SCALAR)
totalbrickdur += dur
chordsdurs_brick.append([cval, dur])
chordsdurs.append(chordsdurs_brick)
rcdlist.append(chordsdurs)
#################### Parse features for each leadsheet
roadmap_counter = -1
for filename in os.listdir(ldir):
"""
Parsing strategy:
Want these features: clist, mlist,poslist,ckeylist,namelist,dlist,splist
1. clist: chords, 1 per note
2. mlist: note pitches
3. poslist: list of (low,high) relative pitch positions
4. ckeylist: list of pitches in the chord
5. namelist: list of filenames
6. dlist: list of durations (or beat positions)
7. splist: list of starting note tuples (pitch, duration, beat pos, chord key, other dur)
8. alist: list of attacks
Parsing strategy:
1. Parse leadsheet (segmented into 48 timeslots) into c (chord keys and notes), m (notes).
2. Calculate pos for each m
3.
"""
roadmap_counter += 1
chordsdurs = rcdlist[roadmap_counter]
fdir = ldir + filename
try:
# Parse leadsheet
c, m = leadsheet.parse_leadsheet(fdir)
except KeyError:
if verbose:
print("KEY ERROR: " + filename)
keyerror_count += 1
continue
except AssertionError:
if verbose:
print("ASSERT ERROR: " + filename)
asserterror_count += 1
continue
try:
totalnotedur = 0
for note in m:
totalnotedur += note[1]
brick_durcount = 0
actual_chordsdurs = []
brick_index = 0
while brick_durcount <= totalnotedur:
chordsdurs_brick = chordsdurs[brick_index % len(chordsdurs)]
brick_index += 1
actual_chordsdurs_brick = []
for j in range(len(chordsdurs_brick)):
chorddur = chordsdurs_brick[j]
brick_dur = int(chorddur[1])
brick_c = chorddur[0]
if brick_c != c[brick_durcount % len(c)]:
continue
brick_durcount += brick_dur
actual_chordsdurs_brick.append(chorddur)
actual_chordsdurs.append(actual_chordsdurs_brick)
# Segment sequences by brick endtimes
brick_endtimes = []
timecount = 0
for chordsdurs_brick in actual_chordsdurs:
brick_dur = 0
for chorddur in chordsdurs_brick:
brick_dur += int(chorddur[1])
pass
timecount += int(brick_dur)
brick_endtimes.append(timecount)
pass
print("Num bricks: ", len(brick_endtimes))
print("Last endtime: ", brick_endtimes[-1])
brick_count = 0 # Tracks brick count across leadsheet
# Repeat chords until they match the notes for each timestep
beat_positions = []
attacks = []
pitches = []
pitches_skip = []
clen = len(c)
totaldur = 0
seq_noskip = []
seq_skip = []
seqs_noskip = []
seqs_skip = []
for note in m:
#print(str(brick_count) + " " + str(len(seqs_skip)))
pitch = note[0] if note[0] != None else MIDI_MAX + 1
pitch -= MIDI_MIN
dur = note[1]
totaldur += dur
# Save general stats
beat_positions.append(
totaldur %
int(constants.WHOLE / constants.RESOLUTION_SCALAR))
attacks.append(1)
pitches_skip.append(pitch)
pitches.append(pitch)
# Save segmented brick info
endtime = brick_endtimes[brick_count]
if totaldur > endtime:
# We need to break this note into subparts to fit the bricks
# End sequence at brick endtime
bp = endtime % int(
constants.WHOLE / constants.RESOLUTION_SCALAR)
if bp in [11, 17, 23, 29]:
print("WUT1", bp)
seq_skip.append([
pitch, endtime %
int(constants.WHOLE / constants.RESOLUTION_SCALAR),
totaldur - dur
])
seq_noskip.append([pitch, 1, totaldur - dur])
for k in range(endtime - (totaldur - dur) - 1):
seq_noskip.append([pitch, 0, totaldur - dur + k + 1])
# Save sequences and resetm
seqs_skip.append(seq_skip)
seqs_noskip.append(seq_noskip)
if max_length < len(seq_skip): # and len(seq_skip) < 30:
max_length = len(seq_skip)
seq_skip = []
seq_noskip = []
brick_count += 1
# Add leftovers
bp = (totaldur - endtime) % int(
constants.WHOLE / constants.RESOLUTION_SCALAR)
if bp in [11, 17, 23, 29]:
print("WUT2", bp)
seq_skip.append([
pitch, (totaldur - endtime) %
int(constants.WHOLE / constants.RESOLUTION_SCALAR),
endtime
])
seq_noskip.append([pitch, 1, endtime])
for k in range(totaldur - endtime - 1):
seq_noskip.append([pitch, 0, endtime + k + 1])
# Update general attacks/pitches
for _ in range(endtime - (totaldur - dur) - 1):
attacks.append(0)
pitches.append(pitch)
attacks.append(1)
pitches.append(pitch)
for _ in range(totaldur - endtime - 1):
attacks.append(0)
pitches.append(pitch)
else:
# Update sequences as normal
bp = totaldur % int(
constants.WHOLE / constants.RESOLUTION_SCALAR)
if bp in [11, 17, 23, 29]:
print("WUT3", bp)
seq_skip.append([
pitch, totaldur %
int(constants.WHOLE / constants.RESOLUTION_SCALAR),
totaldur - dur
])
seq_noskip.append([pitch, 1, totaldur - dur])
for k in range(dur - 1):
seq_noskip.append([pitch, 0, totaldur - dur + k + 1])
# Update general attacks/pitches
for _ in range(dur - 1):
attacks.append(0)
pitches.append(pitch)
if totaldur == endtime:
# Brick finished: Save current sequences and reset
seqs_skip.append(seq_skip)
seqs_noskip.append(seq_noskip)
if max_length < len(seq_skip): # and len(seq_skip) < 30:
max_length = len(seq_skip)
seqdur = 0
if len(seq_skip) > 4:
prevbeatpos = seq_skip[4][1]
for pitch, beatpos, index in seq_skip[4:]:
seqdur += (beatpos - prevbeatpos + 48) % 48
prevbeatpos = beatpos
if len(seq_skip) > 4 and seqdur < 48 * 4:
for pitch, beatpos, index in seq_skip:
note = [pitch]
if note[0] != MIDI_MAX - MIDI_MIN + 1 and note[
0] < lowest_note:
lowest_note = note[0]
if note[0] != MIDI_MAX - MIDI_MIN + 1 and note[
0] > highest_note:
highest_note = note[0]
seq_skip = []
seq_noskip = []
brick_count += 1
for i in range(totaldur - clen):
c.append(c[i % clen])
print("Last index: ", len(c))
sps = []
for i in range(4):
note = m[i]
noteval = MIDI_MAX + 1 - MIDI_MIN if note[
0] == None else note[0] - MIDI_MIN
dur = note[1]
spi = [noteval, beat_positions[i], dur, c[i][0], dur]
if beat_positions[i] in [11, 17, 23, 29]:
print("WUT", beat_positions[i])
sps.append(spi)
# Transpose everything
newc = []
newpitches_skip = []
newp_skip = []
newckeys = []
newfilenames = []
newbeats = []
newsps = []
newattacks = []
newpitches = []
newp = []
transposed_seqs_skip = []
transposed_seqs_noskip = []
for keydiff in range(12):
holder = []
for i in range(len(seqs_skip)):
seq_skip = copy.deepcopy(seqs_skip[i])
for j in range(len(seq_skip)):
note = seq_skip[j][0]
if note != MIDI_MAX - MIDI_MIN + 1:
seq_skip[j][0] = (note + keydiff if
(note + keydiff <= MIDI_MAX -
MIDI_MIN) else note + keydiff -
12)
#if keydiff == 1:
if keydiff < 6:
if note not in possiblepitches.keys():
possiblepitches[note] = 0
possiblepitches[note] += 1
holder.append(seq_skip)
transposed_seqs_skip.append(holder)
holder = []
for i in range(len(seqs_noskip)):
seq_noskip = copy.deepcopy(seqs_noskip[i])
for j in range(len(seq_noskip)):
note = seq_noskip[j][0]
if note != MIDI_MAX - MIDI_MIN + 1:
seq_noskip[j][0] = (note + keydiff if
(note + keydiff <= MIDI_MAX -
MIDI_MIN) else note +
keydiff - 12)
holder.append(seq_noskip)
transposed_seqs_noskip.append(holder)
newctemp = [((x[0] + keydiff) % 12, x[1]) for x in c]
newc.append(newctemp)
newckeys.append([ctuple[0] for ctuple in newctemp])
newfilenames.append(filename)
newbeats.append(beat_positions)
newattacks.append(attacks)
newpitches_skiptemp = [
((note + keydiff if
(note + keydiff <= MIDI_MAX - MIDI_MIN) else note +
keydiff -
12) if note != MIDI_MAX - MIDI_MIN + 1 else note)
for note in pitches_skip
]
newpitchestemp = [
((note + keydiff if
(note + keydiff <= MIDI_MAX - MIDI_MIN) else note +
keydiff -
12) if note != MIDI_MAX - MIDI_MIN + 1 else note)
for note in pitches
]
newpitches_skip.append(newpitches_skiptemp)
newpitches.append(newpitchestemp)
newp_skip.append([
((0.0, 0.0) if (note == MIDI_MAX - MIDI_MIN + 1) else
(float(note / float(MIDI_MAX - MIDI_MIN)),
1.0 - float(note / float(MIDI_MAX - MIDI_MIN))))
for note in newpitches_skiptemp
])
newp.append([((0.0, 0.0) if
(note == MIDI_MAX - MIDI_MIN + 1) else
(float(note / float(MIDI_MAX - MIDI_MIN)),
1.0 - float(note / float(MIDI_MAX - MIDI_MIN))))
for note in newpitchestemp])
newspstemp = []
for i in range(4):
newsp = sps[i]
if newsp[0] != MIDI_MAX - MIDI_MIN + 1:
newsp[0] = sps[i][0] + keydiff if (
sps[i][0] + keydiff <= MIDI_MAX -
MIDI_MIN) else sps[i][0] + keydiff - 12
newsp[3] = (sps[i][3] + keydiff) % 12
newspstemp.append(newsp)
newsps.append(newspstemp)
clist.append(newc)
mlist.append(newpitches_skip)
poslist.append(newp_skip)
ckeylist.append([ctuple[0] for ctuple in newc])
namelist.append(filename)
for bp in beat_positions:
if bp in [11, 17, 23, 29]:
print("WUT", bp)
dlist.append(beat_positions)
splist.append(newsps)
alist.append(attacks)
mlist_noskip.append(newpitches)
poslist_noskip.append(newp)
tlist_skip.append(transposed_seqs_skip)
tlist_noskip.append(transposed_seqs_noskip)
numParsed += len(seqs_skip)
except KeyError:
print(filename)
if verbose:
print("Num key errors: " + str(keyerror_count))
print("Num assert errors: " + str(asserterror_count))
#print("Num bad base phrases (short, long, error): " + str(bigrest_count))
print("Num phrases successfully parsed: " + str(numParsed))
print("Max length: " + str(max_length))
print("Highest note: " + str(highest_note))
print("Lowest note: " + str(lowest_note))
keys = list(possiblepitches.keys())
keys.sort()
for key in keys:
print(key, possiblepitches[key])
# Thus far, we have parsed the leadsheets from transcriptions. We now parse the leadsheets from ii-V-Is.
#TODO
asserterror_count = 0
keyerror_count = 0
bigrest_count = 0
numParsed = 0
max_length = 0 # For tracking purposes, usually 24 is good
lowest_note = 200 # For tracking purposes
highest_note = 0 # For tracking purposes
possiblepitches = {}
for filename in os.listdir(iiVIldir):
fdir = iiVIldir + filename
try:
# Parse leadsheet
c, m = leadsheet.parse_leadsheet(fdir)
except KeyError:
if verbose:
print("KEY ERROR: " + filename)
keyerror_count += 1
continue
except AssertionError:
if verbose:
print("ASSERT ERROR: " + filename)
asserterror_count += 1
continue
try:
# Repeat chords until they match the notes for each timestep
beat_positions = []
attacks = []
pitches = []
pitches_skip = []
clen = len(c)
totaldur = 0
seq_noskip = []
seq_skip = []
seqs_noskip = []
seqs_skip = []
for note in m:
pitch = note[0] if note[0] != None else MIDI_MAX + 1
pitch -= MIDI_MIN
dur = note[1]
totaldur += dur
# Save general stats
beat_positions.append(
totaldur %
int(constants.WHOLE / constants.RESOLUTION_SCALAR))
attacks.append(1)
pitches_skip.append(pitch)
pitches.append(pitch)
bp = totaldur % int(
constants.WHOLE / constants.RESOLUTION_SCALAR)
if bp in [11, 17, 23, 29]:
print("WUT3", bp)
seq_skip.append([
pitch, totaldur %
int(constants.WHOLE / constants.RESOLUTION_SCALAR),
totaldur - dur
])
seq_noskip.append([pitch, 1, totaldur - dur])
for k in range(dur - 1):
seq_noskip.append([pitch, 0, totaldur - dur + k + 1])
# Update general attacks/pitches
for _ in range(dur - 1):
attacks.append(0)
pitches.append(pitch)
seqs_skip.append(seq_skip)
seqs_noskip.append(seq_noskip)
if max_length < len(seq_skip): # and len(seq_skip) < 30:
max_length = len(seq_skip)
seqdur = 0
if len(seq_skip) > 4:
prevbeatpos = seq_skip[4][1]
for pitch, beatpos, index in seq_skip[4:]:
seqdur += (beatpos - prevbeatpos + 48) % 48
prevbeatpos = beatpos
if len(seq_skip) > 4 and seqdur < 48 * 4:
for pitch, beatpos, index in seq_skip:
note = [pitch]
if note[0] != MIDI_MAX - MIDI_MIN + 1 and note[
0] < lowest_note:
lowest_note = note[0]
if note[0] != MIDI_MAX - MIDI_MIN + 1 and note[
0] > highest_note:
highest_note = note[0]
for i in range(totaldur - clen):
c.append(c[i % clen])
print("Last index: ", len(c))
sps = []
for i in range(4):
note = m[i]
noteval = MIDI_MAX + 1 - MIDI_MIN if note[
0] == None else note[0] - MIDI_MIN
dur = note[1]
spi = [noteval, beat_positions[i], dur, c[i][0], dur]
if beat_positions[i] in [11, 17, 23, 29]:
print("WUT", beat_positions[i])
sps.append(spi)
# Transpose everything
newc = []
newpitches_skip = []
newp_skip = []
newckeys = []
newfilenames = []
newbeats = []
newsps = []
newattacks = []
newpitches = []
newp = []
transposed_seqs_skip = []
transposed_seqs_noskip = []
for keydiff in range(12):
holder = []
for i in range(len(seqs_skip)):
seq_skip = copy.deepcopy(seqs_skip[i])
for j in range(len(seq_skip)):
note = seq_skip[j][0]
if note != MIDI_MAX - MIDI_MIN + 1:
seq_skip[j][0] = (note + keydiff if
(note + keydiff <= MIDI_MAX -
MIDI_MIN) else note + keydiff -
12)
#if keydiff == 1:
if keydiff < 6:
if note not in possiblepitches.keys():
possiblepitches[note] = 0
possiblepitches[note] += 1
holder.append(seq_skip)
transposed_seqs_skip.append(holder)
holder = []
for i in range(len(seqs_noskip)):
seq_noskip = copy.deepcopy(seqs_noskip[i])
for j in range(len(seq_noskip)):
note = seq_noskip[j][0]
if note != MIDI_MAX - MIDI_MIN + 1:
seq_noskip[j][0] = (note + keydiff if
(note + keydiff <= MIDI_MAX -
MIDI_MIN) else note +
keydiff - 12)
holder.append(seq_noskip)
transposed_seqs_noskip.append(holder)
newctemp = [((x[0] + keydiff) % 12, x[1]) for x in c]
newc.append(newctemp)
newckeys.append([ctuple[0] for ctuple in newctemp])
newfilenames.append(filename)
newbeats.append(beat_positions)
newattacks.append(attacks)
newpitches_skiptemp = [
((note + keydiff if
(note + keydiff <= MIDI_MAX - MIDI_MIN) else note +
keydiff -
12) if note != MIDI_MAX - MIDI_MIN + 1 else note)
for note in pitches_skip
]
newpitchestemp = [
((note + keydiff if
(note + keydiff <= MIDI_MAX - MIDI_MIN) else note +
keydiff -
12) if note != MIDI_MAX - MIDI_MIN + 1 else note)
for note in pitches
]
newpitches_skip.append(newpitches_skiptemp)
newpitches.append(newpitchestemp)
newp_skip.append([
((0.0, 0.0) if (note == MIDI_MAX - MIDI_MIN + 1) else
(float(note / float(MIDI_MAX - MIDI_MIN)),
1.0 - float(note / float(MIDI_MAX - MIDI_MIN))))
for note in newpitches_skiptemp
])
newp.append([((0.0, 0.0) if
(note == MIDI_MAX - MIDI_MIN + 1) else
(float(note / float(MIDI_MAX - MIDI_MIN)),
1.0 - float(note / float(MIDI_MAX - MIDI_MIN))))
for note in newpitchestemp])
newspstemp = []
for i in range(4):
newsp = sps[i]
if newsp[0] != MIDI_MAX - MIDI_MIN + 1:
newsp[0] = sps[i][0] + keydiff if (
sps[i][0] + keydiff <= MIDI_MAX -
MIDI_MIN) else sps[i][0] + keydiff - 12
newsp[3] = (sps[i][3] + keydiff) % 12
newspstemp.append(newsp)
newsps.append(newspstemp)
clist.append(newc)
mlist.append(newpitches_skip)
poslist.append(newp_skip)
ckeylist.append([ctuple[0] for ctuple in newc])
namelist.append(filename)
for bp in beat_positions:
if bp in [11, 17, 23, 29]:
print("WUT", bp)
dlist.append(beat_positions)
splist.append(newsps)
alist.append(attacks)
mlist_noskip.append(newpitches)
poslist_noskip.append(newp)
tlist_skip.append(transposed_seqs_skip)
tlist_noskip.append(transposed_seqs_noskip)
numParsed += len(seqs_skip)
except KeyError:
print(filename)
if verbose:
print("Num key errors: " + str(keyerror_count))
print("Num assert errors: " + str(asserterror_count))
#print("Num bad base phrases (short, long, error): " + str(bigrest_count))
print("Num phrases successfully parsed: " + str(numParsed))
print("Max length: " + str(max_length))
print("Highest note: " + str(highest_note))
print("Lowest note: " + str(lowest_note))
keys = list(possiblepitches.keys())
keys.sort()
for key in keys:
print(key, possiblepitches[key])
return {
"full_chords": clist,
"pitches_skip": mlist,
"pos_skip": poslist,
"chord_keys": ckeylist,
"namelist": namelist,
"durs": dlist,
"starts": splist,
"attacks": alist,
"pitches_noskip": mlist_noskip,
"pos_noskip": poslist_noskip,
"transposed_seqs_skip": tlist_skip,
"transposed_seqs_noskip": tlist_noskip
}
def parseLeadsheets(ldir, verbose=False):
"""
Parse a directory of leadsheets into a list of chords and melodies.
Inputs:
tdir: Path to directory containing leadsheets, i.e. "/transcriptions".
"""
asserterror_count = 0
keyerror_count = 0
bigrest_count = 0
numParsed = 0
clist = []
mlist = []
poslist = []
ckeylist = []
namelist = []
if not ldir.endswith("/"):
ldir += "/"
if PITCH_REP == PITCH_MIDI:
print("PITCH: MIDI")
elif PITCH_REP == PITCH_INTERVAL:
print("PITCH: INTERVAL")
elif PITCH_REP == PITCH_CHORD:
print("PITCH: CHORD")
for filename in os.listdir(ldir):
fdir = ldir + filename
try:
c, m = leadsheet.parse_leadsheet(fdir)
mseq = []
pseq = []
cseq = []
index_count = 0
valid_leadsheet = True
slot = 0
if PITCH_REP == PITCH_MIDI:
for note in m:
cseq.append(
c[index_count][0]) # get chord key for the slot
index_count += 1
if note[0] != None:
assert note[0] >= MIDI_MIN
restVal = MIDI_MAX + 1
isRest = note[0] == None
nval = restVal if isRest else note[0]
if isRest and note[1] > 12 or note[1] > 24:
valid_leadsheet = False
break
actNoteVal = nval - MIDI_MIN # scale down to start at 0
pval_low = 0.0 if isRest else float(actNoteVal) / float(
MIDI_MAX - MIDI_MIN)
pval_high = 0.0 if isRest else 1 - pval_low
pseq.append((pval_high, pval_low))
mseq.append((actNoteVal, 0)) # attack new note
for _ in range(note[1] - 1):
pseq.append((pval_high, pval_low))
mseq.append(
(actNoteVal, 1)) # sustain for rest of duration
cseq.append(c[index_count][0])
index_count += 1
if index_count >= SEQ_LEN:
break
elif PITCH_REP == PITCH_INTERVAL:
isStart = True
for note in m:
if isStart and note[0] != None:
prevVal = note[0] - 1
isStart = False
cseq.append(
c[index_count][0]) # get chord key for the slot
index_count += 1
restVal = MAX_INTERV + 1
isRest = note[0] == None
nval = restVal if isRest else note[0] - prevVal
nval = nval - MIN_INTERV
prevVal = prevVal if isRest else note[0]
if isRest and note[1] > 12 or note[1] > 24:
valid_leadsheet = False
break
midival = MIDI_MAX + 1 if isRest else note[0]
pval_low = 0.0 if isRest else float(
midival - MIDI_MIN) / float(MIDI_MAX - MIDI_MIN)
pval_high = 0.0 if isRest else 1 - pval_low
pseq.append((pval_high, pval_low))
mseq.append((nval, 0))
for _ in range(note[1] - 1):
pseq.append((pval_high, pval_low))
mseq.append((0 - MIN_INTERV, 1)) #sustain
cseq.append(c[index_count][0])
index_count += 1
if index_count >= SEQ_LEN:
break
elif PITCH_REP == PITCH_CHORD:
for note in m:
ckey = c[index_count][0]
cseq.append(ckey) # get chord key for the slot
index_count += 1
restVal = 12 # pitches go 0-11
isRest = note[0] == None
nval = restVal if isRest else (note[0] - ckey) % 12
if isRest and note[1] > 12 or note[1] > 24:
valid_leadsheet = False
break
midival = MIDI_MAX + 1 if isRest else note[0]
pval_low = 0.0 if isRest else float(
midival - MIDI_MIN) / float(MIDI_MAX - MIDI_MIN)
pval_high = 0.0 if isRest else 1 - pval_low
pseq.append((pval_high, pval_low))
mseq.append((nval, 0))
for _ in range(note[1] - 1):
pseq.append((pval_high, pval_low))
mseq.append((nval, 1))
cseq.append(c[index_count][0])
index_count += 1
if index_count >= SEQ_LEN:
break
if not valid_leadsheet:
bigrest_count += 1
continue
numParsed += 1
clist.append(c)
mlist.append(mseq)
poslist.append(pseq)
ckeylist.append(cseq)
namelist.append(filename)
except KeyError:
if verbose:
print("KEY ERROR: " + filename)
keyerror_count += 1
except AssertionError:
if verbose:
print("ASSERT ERROR: " + filename)
asserterror_count += 1
if verbose:
print("Num key errors: " + str(keyerror_count))
print("Num assert errors: " + str(asserterror_count))
print("Num leadsheets with too big rests: " + str(bigrest_count))
print("Num leadsheets successfully parsed: " + str(numParsed))
return [clist, mlist, poslist, ckeylist, namelist]
def main(modeltype, batch_size, iterations, learning_rate, segment_len, segment_step, train_save_params, dataset=["dataset"], outputdir="output", validation=None, validation_generate_ct=1, resume=None, resume_auto=False, check_nan=False, generate=False, generate_over=None, auto_connectome_keys=None, **model_kwargs):
generate = generate or (generate_over is not None)
should_setup = not generate
unroll_batch_num = None if generate else training.BATCH_SIZE
for dataset_dir in dataset:
if os.path.samefile(dataset_dir,outputdir):
print("WARNING: Directory {} passed as both dataset and output directory!".format(outputdir))
print("This may cause problems by adding generated samples to the dataset directory.")
while True:
result = input("Continue anyway? (y/n)")
if result == "y":
break
elif result == "n":
sys.exit(0)
else:
print("Please type y or n")
if generate_over is None:
training.set_params(batch_size, segment_step, segment_len)
leadsheets = [training.filter_leadsheets(training.find_leadsheets(d)) for d in dataset]
else:
# Don't bother loading leadsheets, we don't need them
leadsheets = []
if validation is not None:
validation_leadsheets = training.filter_leadsheets(training.find_leadsheets(validation))
else:
validation_leadsheets = None
m = builders[modeltype].build(should_setup, check_nan, unroll_batch_num, **model_kwargs)
m.set_learning_rate(learning_rate)
if resume_auto:
paramfile = os.path.join(outputdir,'final_params.p')
if os.path.isfile(paramfile):
with open(os.path.join(outputdir,'data.csv'), 'r') as f:
for line in f:
pass
lastline = line
start_idx = lastline.split(',')[0]
print("Automatically resuming from {} after iteration {}.".format(paramfile, start_idx))
resume = (start_idx, paramfile)
else:
print("Didn't find anything to resume. Starting from the beginning...")
if resume is not None:
start_idx, paramfile = resume
start_idx = int(start_idx)
m.params = pickle.load( open(paramfile, "rb" ) )
else:
start_idx = 0
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if generate:
print("Setting up generation")
m.setup_produce()
print("Starting to generate")
start_time = time.process_time()
if generate_over is not None:
source, divwidth = generate_over
if divwidth == 'full':
divwidth = 0
elif divwidth == 'debug_firststep':
divwidth = -1
elif len(divwidth)>3 and divwidth[-3:] == 'bar':
divwidth = int(divwidth[:-3])*(constants.WHOLE//constants.RESOLUTION_SCALAR)
else:
divwidth = int(divwidth)
ch,mel = leadsheet.parse_leadsheet(source)
lslen = leadsheet.get_leadsheet_length(ch,mel)
if divwidth == 0:
batch = ([ch],[mel]), [source]
elif divwidth == -1:
slices = [leadsheet.slice_leadsheet(ch,mel,0,1)]
batch = list(zip(*slices)), [source]
else:
slices = [leadsheet.slice_leadsheet(ch,mel,s,s+divwidth) for s in range(0,lslen,divwidth)]
batch = list(zip(*slices)), [source]
training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True, batch=batch)
else:
training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True)
end_time = time.process_time()
print("Generation took {} seconds.".format(end_time-start_time))
else:
training.train(m, leadsheets, iterations, outputdir, start_idx, train_save_params, validation_leadsheets=validation_leadsheets, validation_generate_ct=validation_generate_ct, auto_connectome_keys=auto_connectome_keys)
pickle.dump( m.params, open( os.path.join(outputdir, "final_params.p"), "wb" ) )