def __init__(self,
maxOrder=24,
viewPoints=["pitch", "length"],
dataTrain=None,
dataTrial=None,
stm=True,
evolutive=False):
# viewpoints to use for the model
self.viewPoints = viewPoints
# maximal order for the markov chains
self.maxOrder = maxOrder
# wether we also use short term model or not
self.stm = stm
self.evolutive = evolutive
# list of all models for each viewpoints
self.LTM = []
for viewPoint in self.viewPoints:
self.LTM.append(
longTermModel.longTermModel(viewPoint,
maxOrder=self.maxOrder,
evolutive=evolutive))
def test_getLikelihood(self):
"""
Return the likelihood of a note given a state
:param state: a sequence of viewPoints of sier order
:param note: integer or name of the note
:type state: np.array(order)
:type note: int or string
:return: float value of the likelihood
"""
X = np.arange(1000) % 10
for i in range(1, N):
M = longTermModel.longTermModel("pitch", i)
M.train(X)
alphabet = []
for model in M.models:
alphabet.extend(model.alphabet)
alphabet = list(set(alphabet))
alphabet.sort()
for state in alphabet:
for note in alphabet:
if (int(state) + 1) % 10 == int(note) % 10:
self.assertEqual(M.getLikelihood([int(state)], note),
1.0)
else:
self.assertEqual(M.getLikelihood([int(state)], note),
0.0)
def test_generate(self):
"""
Implement a very easy random walk in order to generate a sequence
:param length: length of the generated sequence
:type length: int
:return: sequence (np.array())
"""
X = []
for i in range(10):
X.append(np.arange(300) % 10)
for order in range(1, N):
M = longTermModel.longTermModel("pitch", order)
M.train(X)
S = M.generate(400)
S.sort()
target = list(np.sort(np.arange(400) % 10))
self.assertEqual(S, target)
#unittest.main()
def eval(self, data, k_fold=1):
Likelihood = []
for i in range(len(data.getData(self.viewPoints[0])) // k_fold):
# We initialize the models
self.LTM = []
for viewPoint in self.viewPoints:
self.LTM.append(
longTermModel.longTermModel(viewPoint,
maxOrder=self.maxOrder))
# We train them with the given dataset
k = 0
for viewPoint in self.viewPoints:
self.LTM[k].train(
data.getData(viewPoint)[:i * k_fold] +
data.getData(viewPoint)[(i + 1) * k_fold:])
print(data.getData(viewPoint))
print()
print(
data.getData(viewPoint)[:i * k_fold] +
data.getData(viewPoint)[(i + 1) * k_fold:])
quit()
k += 1
def test_saveAndLoad(self):
"""
Check wether the loaded object is the same as the saved one
"""
for i in range(1, N):
M1 = longTermModel.longTermModel("pitch", i)
X = np.arange(500) % 10
M1.train(X)
M1.save("longterm.s")
M2 = longTermModel.longTermModel("pitch", 1)
M2.load("longterm.s")
os.remove("longterm.s")
for i in range(len(M1.models)):
self.assertEqual(M1.models[i].__dict__, M2.models[i].__dict__)
def setUp(self):
"""
Construct some models for testing
"""
self.models = []
for i in range(1, N):
for viewPoint in viewPoints:
self.models.append(longTermModel.longTermModel(viewPoint, i))
def test_sample(self):
X = np.arange(1000) % 10
for order in range(2, N):
M = longTermModel.longTermModel("pitch", order)
M.train(X)
for z in M.models[order - 2].stateAlphabet:
state = ast.literal_eval(z)
s = M.sample(state)
self.assertEqual(round(M.getLikelihood(state, s), 2), 1.0)
def cleanWeights(self, order=None):
"""
Delete all trained models and fix an order if given
"""
if order is None:
order = self.maxOrder
self.LTM = []
for viewPoint in self.viewPoints:
self.LTM.append(
longTermModel.longTermModel(viewPoint, maxOrder=self.maxOrder))
def test_train(self):
"""
Fill the matrix from data
:param data: pre-processed data to train with
:type data: data object
"""
X = []
for i in range(10):
X.append(np.arange(200) % 10 - 1)
np.random.shuffle(X[i])
for i in range(N):
M = longTermModel.longTermModel("pitch", i)
M.train(X)
if i == 0:
M = longTermModel.longTermModel("pitch")
M.train(X)
for start in range(len(X) - 2 * N):
for end in range(start + i, len(X) - N):
alphabet = []
for model in M.models:
alphabet.extend(model.alphabet)
alphabet = list(set(alphabet))
alphabet.sort()
p = 0
for z in alphabet:
p += M.getLikelihood(X[start:end], z)
if round(p, 2) != 1:
print(p, X[start:end])
self.assertEqual(round(p, 2), 1.0)
def test_getPrediction(self):
"""
Return the probability distribution of notes from a given state
:param state: a sequence of viewPoints of sier order
:type state: np.array(order)
:return: np.array(alphabetSize).astype(float)
"""
X = np.arange(1000) % 10
for i in range(1, 5):
M = longTermModel.longTermModel("pitch", i)
M.train(X)
state = [1, 2, 3, 4, 5]
for model in M.models:
self.assertEqual(
model.getPrediction(state[-model.order:])['6'], 1.0)
self.assertEqual(M.getPrediction(state)['6'], 1.0)
def __init__(self,
maxOrder=None,
viewPoints=["pitch", "length"],
dataTrain=None,
dataTrial=None,
jump=False,
maxDepth=10,
stm=True):
# viewpoints to use for the model
self.viewPoints = viewPoints
# maximal order for the markov chains
self.maxOrder = maxOrder
#maximal depth for the jump model
self.maxDepth = maxDepth
# we store wether we use jump
self.jump = jump
# wether we also use short term model or not
self.stm = stm
# list of all models for each viewpoints
self.LTM = []
for viewPoint in self.viewPoints:
if self.jump is False:
self.LTM.append(
longTermModel.longTermModel(viewPoint,
maxOrder=self.maxOrder))
else:
self.LTM.append(
jumpModel.jumpModel(viewPoint,
maxOrder=self.maxOrder,
maxDepth=self.maxDepth))
def getDistributionsfromFile(self,
file,
threshold,
short_term_only=False,
long_term_only=False,
normalization=True):
"""
Return likelihood over a score
:param folder: file to compute likelihood on
:type data: string
:return: np.array(length)
"""
D = data.data()
D.addFile(file)
distribution = []
for model in self.LTM:
if model.viewPoint == "length":
dat = D.getData(model.viewPoint)[0]
STM = longTermModel.longTermModel(model.viewPoint,
maxOrder=20,
STM=True,
init=dat)
for i in tqdm(range(1, len(dat))):
# we instanciate a Short Term Model for the current viewpoint
STM.train([dat[:i]], shortTerm=True)
predictions_LTM = model.getPrediction(dat[:i])
predictions_STM = STM.getPrediction(dat[:i])
durations = []
for duration in predictions_LTM:
if duration not in durations and predictions_LTM[
duration] != 0:
durations.append(duration)
for duration in predictions_STM:
if duration not in durations and predictions_STM[
duration] != 0:
durations.append(duration)
distribution_note = {}
for duration in durations:
if duration in predictions_LTM:
p1 = predictions_LTM[duration]
flag = True
else:
p1 = 1 / 30
flag = None
if duration in predictions_STM:
p2 = predictions_STM[duration]
else:
p2 = None
if self.stm and p2 is not None:
if flag is not None:
p = self.mergeProbas([p1, p2], [
model.getRelativeEntropy(dat[:i]),
STM.getRelativeEntropy(dat[:i])
])
else:
p = p2
else:
p = p1
if long_term_only:
p = p1
if short_term_only:
p = p2
if p is None:
p = 1 / 30
distribution_note[duration] = p
distribution.append(distribution_note)
### Time Representation
D = data.data()
D.addFile(file)
probas, entropies = self.getLikelihoodfromFile(
file,
short_term_only=short_term_only,
long_term_only=short_term_only)
# We compute the surprise by using -log2(probas)
probas = -np.log(probas + sys.float_info.epsilon) / np.log(2)
# We get the length of the notes
lengths = D.getData("length")[0]
ret = []
for i in range(len(probas)):
ret.append(probas[i])
for j in range(int(lengths[i])):
ret.append(0)
notes_surprise = ret
indexes = []
probas = []
current_index = 1
for i in range(len(distribution)):
sum_distribution = sum(distribution[i].values())
keys = np.array(list(distribution[i])).astype(int)
keys.sort()
for duration in keys:
duration = str(duration)
if int(duration) < int(
lengths[i]
) and distribution[i][duration] / sum_distribution > threshold:
indexes.append(current_index + int(duration))
probas.append(distribution[i][duration] / sum_distribution)
if normalization:
sum_distribution -= distribution[i][duration]
current_index += int(lengths[i]) + 1
missing_notes = np.zeros(len(notes_surprise))
missing_notes[indexes] = probas
plt.plot(notes_surprise)
plt.plot(missing_notes)
plt.legend(["surprise", "missing notes"])
plt.show()
return notes_surprise, missing_notes
def getLikelihoodfromFile(self,
file,
short_term_only=False,
long_term_only=False):
"""
Return likelihood over a score
:param folder: file to compute likelihood on
:type data: string
:return: np.array(length)
"""
D = data.data()
D.addFile(file)
probas = np.ones(D.getSizeofPiece(0))
probas[0] = 1 / len(self.LTM[0].models[0].alphabet)
entropies = np.zeros(D.getSizeofPiece(0))
L = np.ones(len(self.LTM[0].models[0].alphabet)) / len(
self.LTM[0].models[0].alphabet)
entropies[0] = -np.sum(L * np.log2(L))
for model in self.LTM:
dat = D.getData(model.viewPoint)[0]
if long_term_only is False:
STM = longTermModel.longTermModel(model.viewPoint,
maxOrder=20,
STM=True,
init=dat)
for i in range(1, len(dat)):
# we instanciate a Short Term Model for the current viewpoint
if long_term_only is False:
STM.train([dat[:i]], shortTerm=True)
p1 = model.getLikelihood(dat[:i], dat[i])
if p1 is None:
e1 = 4.9
else:
e1 = model.getEntropy(dat[:i])
flag = True
# This happens when the state never happened in the training data
if p1 is None:
p1 = 1 / 30
e1 = 4.9
flag = None
if long_term_only is False:
p2 = STM.getLikelihood(dat[:i], dat[i])
if p2 is None:
e2 = 4.9
else:
e2 = STM.getEntropy(dat[:i])
if long_term_only:
p = p1
e = e1
elif short_term_only:
p = p2
e = e2
if p is None:
p = 1 / 30
e = 4.9
elif self.stm and p2 is not None:
if flag is not None:
p = self.mergeProbas([p1, p2], [
model.getRelativeEntropy(dat[:i]),
STM.getRelativeEntropy(dat[:i])
])
e = self.mergeProbas([e1, e2], [
model.getRelativeEntropy(dat[:i]),
STM.getRelativeEntropy(dat[:i])
])
else:
p = p2
e = e2
else:
p = p1
e = e1
probas[i] *= p
entropies[i] += e
return probas, entropies
def getLikelihoodfromFile(self,
file,
short_term_only=False,
long_term_only=False):
"""
Return likelihood over a score
:param folder: file to compute likelihood on
:type data: string
:return: np.array(length)
"""
D = data.data()
D.addFile(file)
probas = np.ones(D.getSizeofPiece(0))
probas[0] = 1 / len(self.LTM[0].models[0].alphabet)
for model in self.LTM:
dat = D.getData(model.viewPoint)[0]
STM = longTermModel.longTermModel(model.viewPoint,
maxOrder=20,
STM=True,
init=dat)
for i in tqdm(range(1, len(dat))):
# we instanciate a Short Term Model for the current viewpoint
STM.train([dat[:i]], shortTerm=True)
p1 = model.getLikelihood(dat[:i], dat[i])
flag = True
# This happens when the state never happened in the training data
if p1 is None:
p1 = 1 / 30
flag = None
p2 = STM.getLikelihood(dat[:i], dat[i])
if self.stm and p2 is not None:
if flag is not None:
p = self.mergeProbas([p1, p2], [
model.getRelativeEntropy(dat[:i]),
STM.getRelativeEntropy(dat[:i])
])
else:
p = p2
else:
p = p1
if long_term_only:
p = p1
if short_term_only:
p = p2
if p is None:
p = 1 / 30
probas[i] *= p
if probas[i] == 563540:
print("LTM:", model.getLikelihood(dat[:i], dat[i]))
print("STM:", p2)
#print("ret:", self.mergeProbas([p, p2], [model.getEntropy(dat[:i]), STM.getEntropy(dat[:i])]))
print()
return probas
import sys
sys.path.append('../')
from idyom import longTermModel
from idyom import data
from idyom import score
import numpy as np
import matplotlib.pyplot as plt
L = longTermModel.longTermModel("pitch", maxOrder=None)
M = data.data()
#M.parse("../dataset/")
M.parse("dataBaseTest/")
L.train(M.getData("pitch"))
G = L.generate(500)
print(G)
s = score.score(G)
s.plot()
s.writeToMidi("exGen.mid")
L.save("longTermModel.save")
