def create_model(): inputShape = p.load('inputShape') outputSize = p.load('outputSize') model = Sequential() model.add(Masking(mask_value=0., input_shape=inputShape)) model.add(LSTM(256, return_sequences=True)) model.add(LSTM(128, return_sequences=True)) model.add(Dense(outputSize, activation='softmax')) return model
def songBatchGenerator(batch_size): num_samples = p.load('numSamples') start = 0 current_line = 0 myfile = open('training_data.csv') reader = csv.reader(myfile, delimiter=',') while True: start = 0 end = start + batch_size i_d = [] o_d = [] myfile.seek(0) current_line = 0 for row in reader: if current_line >= end: #print(np.array(o_d)[0]) yield (np.array(i_d), np.array(o_d)) start = end end = start + batch_size i_d = [] o_d = [] if end > num_samples: end = num_samples - 1 i_d.append(convertW2V(row[1:], window_size)) o_d.append(convertW2V([row[0]], 1)[0]) current_line += 1
def create_model(): model = Sequential() #model.add(Masking(mask_value=0.)) model.add(LSTM(256, return_sequences=True, input_shape=p.load('inputShape'))) model.add(Dropout(.2)) model.add(LSTM(128, return_sequences=True)) model.add(Dropout(.2)) model.add(LSTM(128, return_sequences=False)) model.add(Dropout(.2)) model.add(Dense(output_size, activation='softmax')) return model
def songBatchGenerator(songList, batchSize, stop): start = 0 count = 0 maxLen = p.load('maxLen') lastTime = 0 totalTime = 0 thisTime = 0 while True: currentTime = time.time() if lastTime != 0: thisTime = currentTime - lastTime totalTime += thisTime lastTime = currentTime end = start + batchSize i_d = [] o_d = [] if end > len(songList) - 1: end = len(songList) - 1 tempList = songList[start:end] for song in tempList: i_d.append(convertW2V(song[0:len(song) - 1], maxLen)) o_d.append(convertOneHot(song[1:], encoderDict, maxLen)) yield np.array(i_d), np.array(o_d) print( "********************************************************************************" ) print(count, " ", start, " ", end) print(count / stop * 100, "% Done") if count > 0: print("Estimated Time Remaining: ", totalTime / count * (stop - count) / 60, "minutes") print("Average Time: ", totalTime / count) print("Remaining Iterations: ", (stop - count)) print( "********************************************************************************" ) start = end if start >= len(songList) - 1: start = 0 count += 1 if count == stop: break
from scipy.io import wavfile from scipy.signal import resample import librosa, glob import pandas as pd import numpy as np import ezPickle as p from scipy.misc import imsave df = pd.DataFrame() print("Please enter name of input folder") in_folder = input() print("Please enter name of output folder") out_folder = input() le = p.load('le') data = [] lengths = [] count = 0 files = glob.glob(in_folder + "/*") for file_name in files: print(file_name) rate, data = wavfile.read(file_name) print("\t read") print('\t', data.shape, " Frames at ", rate) print('\t', count / len(files)) #new_data = resample(data, round(data.shape[0]/rate * new_rate)) spectrogram = librosa.feature.melspectrogram(y=data.astype(float), sr=rate) print("\t Converted") print('\t', spectrogram.shape) imsave(out_folder + '/' + file_name[file_name.index('/'):] + ".png", spectrogram) print("\twrote")
from keras.models import Sequential from keras.layers import LSTM, Dense, Dropout, Masking from keras.utils import multi_gpu_model from keras.optimizers import RMSprop from keras.activations import softmax import tensorflow as tf import keras, math import numpy as np import pickle import os, sys, csv import ezPickle as p import time window_size = p.load('window_size') os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3" keras.callbacks.TensorBoard(histogram_freq=0) w2v = p.load('w2v') size = p.load('size') def convertW2V(items, padLength): l = [] for item in items: l.append(w2v[item]) for i in range(padLength - len(items)): l.append(list(np.zeros(20))) return l def convertOneHot(items, dict, padLength): l = []
import math import numpy as np import ezPickle as p w2v = p.load('w2v') size = p.load('size') def convertW2V(items, padLength): l = [] for item in items: l.append(w2v[item]) for i in range(padLength - len(items)): l.append(list(np.zeros(20))) return l def convertOneHot(items, dict, padLength): l = [] for item in items: l.append(dict[str(item)]) for i in range(padLength - len(items)): l.append(np.zeros(len(l[0]))) return l def makeOneHotEncodeDict(corpus): dict = {} count = 0 for item in corpus: if not str(item) in dict.keys(): dict[str(item)] = count count+=1 for key in dict.keys(): my_list = [0]*count my_list[dict[key]] = 1 dict[key] = my_list
from gensim.models import Word2Vec import ezPickle as p size = 20 songs = p.load('songList') w2v = Word2Vec(sg=1, seed=1, size=size, window=8, min_count=0, workers=2) w2v.build_vocab(songs) w2v.train(songs, total_examples=w2v.corpus_count, epochs=100) p.save(w2v, 'w2v') p.save(size, "size") print(len(w2v.wv.vocab))
from keras.models import Sequential from keras.layers import LSTM, Dense, Dropout, Masking from keras.utils import multi_gpu_model from keras.optimizers import RMSprop from keras.activations import softmax import tensorflow as tf import keras, math import numpy as np import pickle import os, sys, csv import ezPickle as p import time window_size = p.load('window_size') os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3" keras.callbacks.TensorBoard(histogram_freq=0) w2v = p.load('w2v') def convertW2V(items, padLength): l = [] for item in items: l.append(w2v[item]) for i in range(padLength - len(items)): l.append(list(np.zeros(20))) return l def convertOneHot(items, dict, padLength): l = [] for item in items:
from keras.models import Sequential from keras.layers import LSTM, Dense, Dropout, Masking from keras.activations import softmax from keras.optimizers import RMSprop from keras.utils import multi_gpu_model import keras import numpy as np import ezPickle as p import math keras.callbacks.TensorBoard(histogram_freq=0) w2v = p.load('w2v') def convertW2V(items, padLength): l = [] for item in items: l.append(w2v[item]) for i in range(padLength - len(items)): l.append(list(np.zeros(20))) return l def makeOneHotEncodeDict(corpus): dict = {} count = 0 for item in corpus: if not str(item) in dict.keys(): dict[str(item)] = count count += 1
import ezPickle as p from keras.models import Sequential from keras.layers import LSTM, Dense, Dropout, Masking from keras.utils import multi_gpu_model from keras.optimizers import RMSprop from keras.activations import softmax import tensorflow as tf import numpy as np import math, keras import os import time keras.callbacks.TensorBoard(histogram_freq=0) w2v = p.load('w2v') os.environ["CUDA_VISIBLE_DEVICES"] = "3" #"1,2,3" def convertW2V(items, padLength): l = [] for item in items: l.append(w2v[item]) #print(w2v[item]) for i in range(padLength - len(items)): l.append(list(np.zeros(20))) return l def convertOneHot(items, mydict, padLength): l = [] for item in items: l.append(mydict[str(item)]) for i in range(padLength - len(items)):
import ezPickle as p import pandas as pd from sklearn import svm series_stats = pd.read_csv('../data/summary_stats.csv') test_stats = pd.read_csv('../data/test_summary_stats.csv') outputs = p.load('output_list') print('load data done..') for i in range(len(outputs)): outputs[i] = outputs[i].index(1) clf = svm.SVC(gamma='scale', decision_function_shape='ovo') print('train...') clf.fit(series_stats.values, outputs) p.save(clf,'clf') print('train done...') predictions = clf.predict(test_stats.values) print('predict done...') def getReverseOneHotDict(dict): inverted_dict = {str(value): key for key, value in dict.items()} return inverted_dict encoder_dict = p.load('encoder_dict') decoder_dict = getReverseOneHotDict(encoder_dict) predictions = predictions.tolist() for i in range(len(predictions)): temp = [0]*9 temp[predictions[i]] = 1 predictions[i] = temp
from scipy.io import wavfile from scipy.signal import resample from keras.preprocessing.sequence import pad_sequences from keras import Sequential from keras import backend as K from keras.layers import Dense, LSTM, Dropout, Masking from keras.optimizers import RMSprop from keras.activations import softmax from keras.utils import multi_gpu_model from sklearn import preprocessing import keras, math, os, glob import pandas as pd import numpy as np import ezPickle as p from scipy.misc import imread le = p.load('le') train_curated = pd.read_csv('train_curated.csv') files = train_curated['fname'] labels = train_curated['labels'] p_model = p.load('p_model') max_len = 1800 correct = 0 for i in range(len(files)): #print(file_name) file_name = files[i] spectrogram = imread("mel_spec_curated/" + file_name + ".png") x = np.array( pad_sequences([np.true_divide(np.transpose((spectrogram)), 256)], maxlen=max_len, value=-1)).reshape(1, max_len, 128)
def dataGenerator(n): for i in range(math.floor(n / 10)): yield (np.array(p.load('inputData' + str(i))), np.array(p.load('outputData' + str(i))))
import ezPickle as p import pandas as pd from sklearn.ensemble import RandomForestClassifier series_stats = pd.read_csv('../data/summary_stats.csv') outputs = p.load('output_list') for i in range(len(outputs)): outputs[i] = outputs[i].index(1) clf = RandomForestClassifier(n_estimators=500, max_depth=7, random_state=0) clf.fit(series_stats.values[0:3000], outputs[0:3000]) p.save(clf, 'rf_clf') print(clf.score(series_stats.values[3000:], outputs[3000:]))
#split the training data into batches import ezPickle as p import math print("loading data") inputData = p.load('inputData') outputData = p.load('outputData') print('Size is : ', len(inputData)) #Split into Sets of 12 songs numSongs = 12 sets = math.floor(len(inputData)/numSongs) tempInput = [] tempOutput = [] #making output numSongs = 12 for i in range(sets): start = i*numSongs end = (i+1)*numSongs tempInput = inputData[start:end] tempOutput = outputData[start:end] p.save(tempInput, 'inputData'+str(i)) p.save(tempOutput, 'outputData'+str(i)) if end >= len(inputData): tempInput = inputData[start:] tempOutput = outputData[start:] p.save(tempInput, 'inputData'+str(i)) p.save(tempOutput, 'outputData'+str(i)) print(inputData[0])
import pickle, music21 import numpy as np import random, os import ezPickle as p maxLen = p.load('maxLen') os.environ["CUDA_VISIBLE_DEVICES"] = "1" #"1,2,3" import math, random, csv window_size = p.load('window_size') w2v = p.load("w2v") def closestVec(vec): newVec = w2v.wv.most_similar([vec], [], 1) return w2v[newVec[0][0]] def randCloseVec(vec): newVecs = w2v.wv.most_similar([vec], [], 10) newVec = [] for i in range(10): newVec = newVecs[i][0] if random.random() < .5: break return w2v[newVec] def convertW2V(items, padLength): l = []
from keras.layers import Dense, Conv2D, Dropout, MaxPooling2D, Flatten from keras.optimizers import RMSprop from keras.utils import multi_gpu_model import math, os, keras import pandas as pd import numpy as np from scipy.misc import imread import ezPickle as p os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3" train_curated = pd.read_csv('train_curated.csv') train_noisy = pd.read_csv('train_noisy.csv') files = train_curated['fname'] labels = train_curated['labels'] n_files = train_noisy['fname'] n_labels = train_noisy['labels'] le = p.load('le') max_len = 0 cb = keras.callbacks.TensorBoard(histogram_freq=0) for i in range(len(files)): file_name = files[i] spectrogram = imread("trimmed_curated_spec/" + file_name + ".png") if max_len < spectrogram.shape[1]: max_len = spectrogram.shape[1] print(max_len) def generator(size): input_batch = [] output_batch = [] for i in range(size): file_name = files[i]