def getSamples(basedir):
X, Y = [], []
feature_labels = [
'segments_pitch', 'segments_timbre', 'segments_loudness_max', 'tempo'
]
cnt = 0
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*.h5'))
print(root, cnt)
# apply function to all files
for f in files:
# try:
h5 = GETTERS.open_h5_file_read(f)
year = GETTERS.get_year(h5)
if year == 0:
continue
bttimbre = get_bttimbre(h5)
if bttimbre is None or year == 0:
continue
h5.close()
X.append(bttimbre)
Y.append(year)
cnt += 1
if cnt > 10000:
break
return X, Y, feature_labels
def buildfeatures(basedir,cluster,ext='.h5'):
global cap
i = 0
features = []
decade = []
decadecount = defaultdict(int)
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
year = hdf5_getters.get_year(h5)
if year == 0:
h5.close()
continue
bins = getbin(year)
# dec = (year/10)*10
# if dec < 1960:
# h5.close()
# continue
if decadecount[bins] > cap:
flag = checkforcompletion(decadecount)
h5.close()
if flag:
return features,decade
continue
i += 1
print i
clustercount = {}
for x in range(50):
clustercount[x] = 0
feature = []
try:
bttimbre = bt.get_bttimbre(h5)
btT = bttimbre.T
for x in btT:
label = kmeans.predict(x)
clustercount[label[0]] += 1
for cl in clustercount.keys():
feature.append(clustercount[cl])
features.append(feature)
decade.append(bins)
decadecount[bins] += 1
except:
h5.close()
continue
h5.close()
return features,decade
def getclusters(basedir,ext='.h5') :
print 'inside clusters'
features = []
cfeatures = []
decadecount = defaultdict(int)
deccount = defaultdict(int)
i=0
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
year = hdf5_getters.get_year(h5)
if year == 0:
h5.close()
continue
bins = getbin(year)
# decade.append(bin)
# dec = (year/10)*10
# decadecount[dec] += 1
# if dec < 1960:
# h5.close()
# continue
deccount[bin] += 1
if decadecount[bins] > cap:
flag = checkforcompletion(decadecount)
h5.close()
if flag:
for dec in decadecount.keys():
print 'Dec : ' + str(dec) + ' Count : ' + str(decadecount[dec])
return features,cfeatures
continue
i += 1
print i
try:
bttimbre = bt.get_bttimbre(h5)
btT = bttimbre.T
for x in btT:
features.append(x)
decadecount[bins] += 1
btchroma = bt.get_btchromas(h5)
btc = btchroma.T
for x in btc:
cfeatures.append(x)
decadecount[bins] += 1
except:
h5.close()
continue
h5.close()
for dec in deccount.keys():
print 'Dec : ' + str(dec) + ' Count : ' + str(decadecount[dec])
features = array(features)
cfeatures = array(cfeatures)
return features,cfeatures
def beat_aligned_timbre(song_id, file_name) :
bttimbre = beat_aligned_feats.get_bttimbre(file_name)
if bttimbre is None :
return []
bttimbre = bttimbre.T
rows = []
for i in range(bttimbre.shape[0]) :
vals = {'song_id': song_id, 'beat_number': i}
for j in range(12) :
vals[timbre_schema[j]] = bttimbre[i][j]
rows.append(vals)
return rows
def process_filelist_test(filelist=None,
model=None,
tmpfilename=None,
npicks=None,
winsize=None,
finaldim=None,
K=1,
typecompress='picks'):
"""
Main function, process all files in the list (as long as their artist
is in testartist)
INPUT
filelist - a list of song files
model - h5 file containing feats and year for all train songs
tmpfilename - where to save our processed features
npicks - number of segments to pick per song
winsize - size of each segment we pick
finaldim - how many values do we keep
K - param of KNN (default 1)
typecompress - feature type, 'picks', 'corrcoeff' or 'cov'
must be the same as in training
"""
# sanity check
for arg in locals().values():
assert not arg is None, 'process_filelist_test, missing an argument, something still None'
if os.path.isfile(tmpfilename):
print 'ERROR: file', tmpfilename, 'already exists.'
return
if not os.path.isfile(model):
print 'ERROR: model', model, 'does not exist.'
return
# create kdtree
h5model = tables.openFile(model, mode='r')
assert h5model.root.data.feats.shape[
1] == finaldim, 'inconsistency in final dim'
kd = ANN.kdtree(h5model.root.data.feats)
# create outputfile
output = tables.openFile(tmpfilename, mode='a')
group = output.createGroup("/", 'data', 'TMP FILE FOR YEAR RECOGNITION')
output.createEArray(group,
'year_real',
tables.IntAtom(shape=()), (0, ),
'',
expectedrows=len(filelist))
output.createEArray(group,
'year_pred',
tables.Float64Atom(shape=()), (0, ),
'',
expectedrows=len(filelist))
# random projection
ndim = 12 # fixed in this dataset
if typecompress == 'picks':
randproj = RANDPROJ.proj_point5(ndim * winsize, finaldim)
elif typecompress == 'corrcoeff' or typecompress == 'cov':
randproj = RANDPROJ.proj_point5(ndim * ndim, finaldim)
elif typecompress == 'avgcov':
randproj = RANDPROJ.proj_point5(90, finaldim)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
# go through files
cnt_f = 0
for f in filelist:
cnt_f += 1
if cnt_f % 5000 == 0:
print 'TESTING FILE #' + str(cnt_f)
# check file
h5 = GETTERS.open_h5_file_read(f)
artist_id = GETTERS.get_artist_id(h5)
year = GETTERS.get_year(h5)
track_id = GETTERS.get_track_id(h5)
h5.close()
if year <= 0: # probably useless but...
continue
if typecompress == 'picks':
# we have a train artist with a song year, we're good
bttimbre = get_bttimbre(f)
if bttimbre is None:
continue
# we even have normal features, awesome!
processed_feats = CBTF.extract_and_compress(bttimbre,
npicks,
winsize,
finaldim,
randproj=randproj)
elif typecompress == 'corrcoeff':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.corr_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'cov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.cov_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'avgcov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.avgcov_and_compress(timbres,
finaldim,
randproj=randproj)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
if processed_feats is None:
continue
if processed_feats.shape[0] == 0:
continue
# do prediction
year_pred = do_prediction(processed_feats, kd, h5model, K)
# add pred and ground truth to output
if not year_pred is None:
output.root.data.year_real.append([year])
output.root.data.year_pred.append([year_pred])
# close output and model
del kd
h5model.close()
output.close()
# done
return
def process_filelist_train(filelist=None,testartists=None,tmpfilename=None,
npicks=None,winsize=None,finaldim=None,typecompress='picks'):
"""
Main function, process all files in the list (as long as their artist
is not in testartist)
INPUT
filelist - a list of song files
testartists - set of artist ID that we should not use
tmpfilename - where to save our processed features
npicks - number of segments to pick per song
winsize - size of each segment we pick
finaldim - how many values do we keep
typecompress - one of 'picks' (win of btchroma), 'corrcoef' (correlation coefficients),
'cov' (covariance)
"""
# sanity check
for arg in locals().values():
assert not arg is None,'process_filelist_train, missing an argument, something still None'
if os.path.isfile(tmpfilename):
print 'ERROR: file',tmpfilename,'already exists.'
return
# create outputfile
output = tables.openFile(tmpfilename, mode='a')
group = output.createGroup("/",'data','TMP FILE FOR YEAR RECOGNITION')
output.createEArray(group,'feats',tables.Float64Atom(shape=()),(0,finaldim),'',
expectedrows=len(filelist))
output.createEArray(group,'year',tables.IntAtom(shape=()),(0,),'',
expectedrows=len(filelist))
output.createEArray(group,'track_id',tables.StringAtom(18,shape=()),(0,),'',
expectedrows=len(filelist))
# random projection
ndim = 12 # fixed in this dataset
if typecompress == 'picks':
randproj = RANDPROJ.proj_point5(ndim * winsize, finaldim)
elif typecompress == 'corrcoeff' or typecompress == 'cov':
randproj = RANDPROJ.proj_point5(ndim * ndim, finaldim)
elif typecompress == 'avgcov':
randproj = RANDPROJ.proj_point5(90, finaldim)
else:
assert False,'Unknown type of compression: '+str(typecompress)
# iterate over files
cnt_f = 0
for f in filelist:
cnt_f += 1
# verbose
if cnt_f % 50000 == 0:
print 'training... checking file #',cnt_f
# check file
h5 = GETTERS.open_h5_file_read(f)
artist_id = GETTERS.get_artist_id(h5)
year = GETTERS.get_year(h5)
track_id = GETTERS.get_track_id(h5)
h5.close()
if year <= 0 or artist_id in testartists:
continue
# we have a train artist with a song year, we're good
bttimbre = get_bttimbre(f)
if typecompress == 'picks':
if bttimbre is None:
continue
# we even have normal features, awesome!
processed_feats = CBTF.extract_and_compress(bttimbre,npicks,winsize,finaldim,
randproj=randproj)
elif typecompress == 'corrcoeff':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.corr_and_compress(timbres,finaldim,randproj=randproj)
elif typecompress == 'cov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.cov_and_compress(timbres,finaldim,randproj=randproj)
elif typecompress == 'avgcov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.avgcov_and_compress(timbres,finaldim,randproj=randproj)
else:
assert False,'Unknown type of compression: '+str(typecompress)
# save them to tmp file
n_p_feats = processed_feats.shape[0]
output.root.data.year.append( np.array( [year] * n_p_feats ) )
output.root.data.track_id.append( np.array( [track_id] * n_p_feats ) )
output.root.data.feats.append( processed_feats )
# we're done, close output
output.close()
return
def process_filelist_train(filelist=None,
testartists=None,
tmpfilename=None,
npicks=None,
winsize=None,
finaldim=None,
typecompress='picks'):
"""
Main function, process all files in the list (as long as their artist
is not in testartist)
INPUT
filelist - a list of song files
testartists - set of artist ID that we should not use
tmpfilename - where to save our processed features
npicks - number of segments to pick per song
winsize - size of each segment we pick
finaldim - how many values do we keep
typecompress - one of 'picks' (win of btchroma), 'corrcoef' (correlation coefficients),
'cov' (covariance)
"""
# sanity check
for arg in locals().values():
assert not arg is None, 'process_filelist_train, missing an argument, something still None'
if os.path.isfile(tmpfilename):
print 'ERROR: file', tmpfilename, 'already exists.'
return
# create outputfile
output = tables.openFile(tmpfilename, mode='a')
group = output.createGroup("/", 'data', 'TMP FILE FOR YEAR RECOGNITION')
output.createEArray(group,
'feats',
tables.Float64Atom(shape=()), (0, finaldim),
'',
expectedrows=len(filelist))
output.createEArray(group,
'year',
tables.IntAtom(shape=()), (0, ),
'',
expectedrows=len(filelist))
output.createEArray(group,
'track_id',
tables.StringAtom(18, shape=()), (0, ),
'',
expectedrows=len(filelist))
# random projection
ndim = 12 # fixed in this dataset
if typecompress == 'picks':
randproj = RANDPROJ.proj_point5(ndim * winsize, finaldim)
elif typecompress == 'corrcoeff' or typecompress == 'cov':
randproj = RANDPROJ.proj_point5(ndim * ndim, finaldim)
elif typecompress == 'avgcov':
randproj = RANDPROJ.proj_point5(90, finaldim)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
# iterate over files
cnt_f = 0
for f in filelist:
cnt_f += 1
# verbose
if cnt_f % 50000 == 0:
print 'training... checking file #', cnt_f
# check file
h5 = GETTERS.open_h5_file_read(f)
artist_id = GETTERS.get_artist_id(h5)
year = GETTERS.get_year(h5)
track_id = GETTERS.get_track_id(h5)
h5.close()
if year <= 0 or artist_id in testartists:
continue
# we have a train artist with a song year, we're good
bttimbre = get_bttimbre(f)
if typecompress == 'picks':
if bttimbre is None:
continue
# we even have normal features, awesome!
processed_feats = CBTF.extract_and_compress(bttimbre,
npicks,
winsize,
finaldim,
randproj=randproj)
elif typecompress == 'corrcoeff':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.corr_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'cov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.cov_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'avgcov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.avgcov_and_compress(timbres,
finaldim,
randproj=randproj)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
# save them to tmp file
n_p_feats = processed_feats.shape[0]
output.root.data.year.append(np.array([year] * n_p_feats))
output.root.data.track_id.append(np.array([track_id] * n_p_feats))
output.root.data.feats.append(processed_feats)
# we're done, close output
output.close()
return
def get_all_titles(basedir,ext='.h5') :
global errorcount
global count
global cap
global truecount
features = []
decade = []
decadecount = defaultdict(int)
timbre = None
i = 0
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
feature = []
try:
h5 = hdf5_getters.open_h5_file_read(f)
except HDF5ExtError as e:
errorcount += 1
print "Unexpected error:", sys.exc_info()[0]
print traceback.format_exc()
continue
year = hdf5_getters.get_year(h5)
print i
i+=1
if year == 0:
h5.close()
continue
label = getbin(year)
#label = (year/10)*10
truecount[label] += 1
if decadecount[label] > cap:
flag = checkforcompletion(decadecount)
h5.close()
if flag:
for dec in decadecount.keys():
print 'Decade : ' + str(dec) + ' Count : ' + str(decadecount[dec])
return features,decade
continue
# dec = (year/10)*10
# if dec < 1960:
# h5.close()
# continue
# try:
#
# bttimbre = bt.get_bttimbre(h5)
# timbres = bttimbre.argmax(axis = 0) + 1 # Is a vector of timbre values sutiable for training an HMM
# for timbre in timbres:
# timbredict[timbre] += 1
# for i in range(1,13):
# feature.append(timbredict[i])
# except:
# h5.close()
# continue
# clustercount = {}
# for x in range(12):
# clustercount[x] = 0
# try:
# bttimbre = bt.get_bttimbre(h5)
# btT = bttimbre.T
# for x in btT:
# timbre = x.argmax(axis = 0)
# clustercount[timbre]+=1
# except:
# h5.close()
# continue
# for y in range(12):
# features.append(clustercount[y])
try:
btchromas = bt.get_btchromas(h5)
for chroma in btchromas:
feature.append(mean(chroma))
covmat = get_covariance(btchromas)
feature.extend(covmat)
bttimbre = bt.get_bttimbre(h5)
for timbre in bttimbre:
feature.append(mean(timbre))
covmat = get_covariance(bttimbre)
feature.extend(covmat)
# btT = bttimbre.T
# for x in btT:
# timbre = x.argmax(axis = 0)
# clustercount[timbre]+=1
# for y in range(12):
# feature.append(clustercount[y])
except:
errorcount += 1
h5.close()
continue
loudness = hdf5_getters.get_loudness(h5)
feature.append(loudness)
duration = hdf5_getters.get_duration(h5)
feature.append(duration)
features.append(feature)
decade.append(label)
decadecount[label] += 1
count += 1
h5.close()
# title = hdf5_getters.get_title(h5)
# segstarts = hdf5_getters.get_segments_start(h5)
# segstarts = np.array(segstarts).flatten()
# btstarts = hdf5_getters.get_beats_start(h5)
# btstarts = np.array(btstarts).flatten()
for dec in decadecount.keys():
print 'Decade : ' + str(dec) + ' Count : ' + str(decadecount[dec])
return features,decade
files, start_times, end_times = load_results(tsv_path)
for filename, start_time, end_time in zip(files, start_times, end_times):
# Load in MSD hdf5 file
h5 = hdf5_getters.open_h5_file_read(to_h5_path(filename))
# Load in beat times from MSD
beats = hdf5_getters.get_beats_start(h5)
# Some files have no EN analysis
if beats.size == 0:
continue
# Get indices which fall within the range of correct alignment
time_mask = np.logical_and(beats > start_time, beats < end_time)
beats = beats[time_mask]
# and beat-synchronous feature matrices, within the time range of correct alignment
chroma = beat_aligned_feats.get_btchromas(h5)[:, time_mask]
timbre = beat_aligned_feats.get_bttimbre(h5)[:, time_mask]
loudness = beat_aligned_feats.get_btloudnessmax(h5)[:, time_mask]
h5.close()
# Stack it
msd_features = np.vstack([chroma, timbre, loudness])
if np.isnan(msd_features).any():
print filename
continue
# Load in pretty midi object
pm = pretty_midi.PrettyMIDI(midi.read_midifile(to_midi_path(filename)))
# Construct piano roll, aligned to the msd beat times
piano_roll = pm.get_piano_roll(times=beats)
# Ignore notes below 36 and above 84
piano_roll = piano_roll[36:84, :]
# Write out
np.save(os.path.join(output_path, filename.replace('.mp3', '-msd.npy')), msd_features)
def process_filelist_test(filelist=None,model=None,tmpfilename=None,
npicks=None,winsize=None,finaldim=None,K=1,
typecompress='picks'):
"""
Main function, process all files in the list (as long as their artist
is in testartist)
INPUT
filelist - a list of song files
model - h5 file containing feats and year for all train songs
tmpfilename - where to save our processed features
npicks - number of segments to pick per song
winsize - size of each segment we pick
finaldim - how many values do we keep
K - param of KNN (default 1)
typecompress - feature type, 'picks', 'corrcoeff' or 'cov'
must be the same as in training
"""
# sanity check
for arg in locals().values():
assert not arg is None,'process_filelist_test, missing an argument, something still None'
if os.path.isfile(tmpfilename):
print 'ERROR: file',tmpfilename,'already exists.'
return
if not os.path.isfile(model):
print 'ERROR: model',model,'does not exist.'
return
# create kdtree
h5model = tables.openFile(model, mode='r')
assert h5model.root.data.feats.shape[1]==finaldim,'inconsistency in final dim'
kd = ANN.kdtree(h5model.root.data.feats)
# create outputfile
output = tables.openFile(tmpfilename, mode='a')
group = output.createGroup("/",'data','TMP FILE FOR YEAR RECOGNITION')
output.createEArray(group,'year_real',tables.IntAtom(shape=()),(0,),'',
expectedrows=len(filelist))
output.createEArray(group,'year_pred',tables.Float64Atom(shape=()),(0,),'',
expectedrows=len(filelist))
# random projection
ndim = 12 # fixed in this dataset
if typecompress == 'picks':
randproj = RANDPROJ.proj_point5(ndim * winsize, finaldim)
elif typecompress == 'corrcoeff' or typecompress=='cov':
randproj = RANDPROJ.proj_point5(ndim * ndim, finaldim)
elif typecompress == 'avgcov':
randproj = RANDPROJ.proj_point5(90, finaldim)
else:
assert False,'Unknown type of compression: '+str(typecompress)
# go through files
cnt_f = 0
for f in filelist:
cnt_f += 1
if cnt_f % 5000 == 0:
print 'TESTING FILE #'+str(cnt_f)
# check file
h5 = GETTERS.open_h5_file_read(f)
artist_id = GETTERS.get_artist_id(h5)
year = GETTERS.get_year(h5)
track_id = GETTERS.get_track_id(h5)
h5.close()
if year <= 0: # probably useless but...
continue
if typecompress == 'picks':
# we have a train artist with a song year, we're good
bttimbre = get_bttimbre(f)
if bttimbre is None:
continue
# we even have normal features, awesome!
processed_feats = CBTF.extract_and_compress(bttimbre,npicks,winsize,finaldim,
randproj=randproj)
elif typecompress == 'corrcoeff':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.corr_and_compress(timbres,finaldim,randproj=randproj)
elif typecompress == 'cov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.cov_and_compress(timbres,finaldim,randproj=randproj)
elif typecompress == 'avgcov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.avgcov_and_compress(timbres,finaldim,randproj=randproj)
else:
assert False,'Unknown type of compression: '+str(typecompress)
if processed_feats is None:
continue
if processed_feats.shape[0] == 0:
continue
# do prediction
year_pred = do_prediction(processed_feats,kd,h5model,K)
# add pred and ground truth to output
if not year_pred is None:
output.root.data.year_real.append( [year] )
output.root.data.year_pred.append( [year_pred] )
# close output and model
del kd
h5model.close()
output.close()
# done
return