def apply_to_all_files(basedir, func=lambda x: x, ext=".h5"):
"""
From a base directory, go through all subdirectories,
find all files with the given extension, apply the
given function 'func' to all of them.
If no 'func' is passed, we do nothing except counting.
INPUT
basedir - base directory of the dataset
func - function to apply to all filenames
ext - extension, .h5 by default
RETURN
number of files
"""
cnt = 0
# iterate over all files in all subdirectories
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, "*" + ext))
# count files
cnt += len(files)
# apply function to all files
for f in files:
h5 = GETTERS.open_h5_file_read(f)
pdb.set_trace()
func(f)
return cnt
def plots(track):
f, axarr = plt.subplots(2, sharex=True)
path = "../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5"
h5 = GETTERS.open_h5_file_read(path)
segments = (GETTERS.get_segments_start(h5))
sections = (GETTERS.get_sections_start(h5))
max_loudness = (GETTERS.get_segments_loudness_max(h5))
loudness = (GETTERS.get_segments_loudness_start(h5))
average_loudness = (max_loudness + loudness) / 2
average_loudness_song = GETTERS.get_loudness(h5)
start_fade_out = GETTERS.get_start_of_fade_out(h5)
end_fade_in = GETTERS.get_end_of_fade_in(h5)
pitches = GETTERS.get_segments_pitches(h5)
h5.close()
axarr[0].set_title('loudness curve for ' + ut.get_track_info(track))
axarr[0].plot(segments, average_loudness, label='Filtered')
axarr[0].axhline(average_loudness_song, color='green')
for section in enumerate(sections):
axarr[0].axvline(section[1], color='red')
axarr[0].axvline(start_fade_out, color='green')
axarr[0].axvline(end_fade_in, color='green')
idx = list()
for section in sections:
dif = segments - section
posdif = np.where(dif >=0)
idx.append(posdif[0][0])
axarr[1].set_title('Chroma values for ' + ut.get_track_info(track))
#axarr[1].set_xticks(idx,sections.astype(int))
extent =[0,segments.shape[0],0,12]
axarr[1].imshow(pitches.transpose(),extent = extent,aspect = 'auto',interpolation='nearest',origin='lower')
plt.show()
def msd_beatchroma(filename):
"""
Get the same beatchroma as Dan
Our filename is the full path
TESTED
"""
nchr=12
# get segments, pitches, beats, loudness
h5 = GETTERS.open_h5_file_read(filename)
pitches = GETTERS.get_segments_pitches(h5).T
loudness = GETTERS.get_segments_loudness_start(h5)
Tsegs = GETTERS.get_segments_start(h5)
Tbeats = GETTERS.get_beats_start(h5)
h5.close()
# sanity checks
if len(Tsegs) < 3 or len(Tbeats) < 2:
return None
# get chroma and apply per segments loudness
Segs = pitches * np.tile(np.power(10., loudness/20.), (nchr, 1))
if Segs.shape[0] < 12 or Segs.shape[1] < 3:
return None
# properly figure time overlaps and weights
C = resample_mx(Segs, Tsegs, Tbeats)
# renormalize columns
n = C.max(axis=0)
return C * np.tile(1./n, (nchr, 1))
def read_hdf5(file_path):
try:
return hdf.open_h5_file_read(file_path)
except:
tok = file_path.split('/')
print 'Skipped: ' + tok[-1]
return None
def getInfo(files):
data = []
build_str = ''
with open(sys.argv[1], 'r') as f:
contents = f.read()
c = contents.split()
f.close()
print("creating csv with following fields:" + contents)
for i in c:
build_str = build_str + i + ','
build_str = build_str[:-1]
build_str = build_str + '\n'
for fil in files:
curFile = getters.open_h5_file_read(fil)
d2 = {}
get_table = {'track_id': getters.get_track_id(curFile), 'segments_pitches': getters.get_segments_pitches(curFile), 'time_signature_confidence': getters.get_time_signature_confidence(curFile), 'song_hotttnesss': getters.get_song_hotttnesss(curFile), 'artist_longitude': getters.get_artist_longitude(curFile), 'tatums_confidence': getters.get_tatums_confidence(curFile), 'num_songs': getters.get_num_songs(curFile), 'duration': getters.get_duration(curFile), 'start_of_fade_out': getters.get_start_of_fade_out(curFile), 'artist_name': getters.get_artist_name(curFile), 'similar_artists': getters.get_similar_artists(curFile), 'artist_mbtags': getters.get_artist_mbtags(curFile), 'artist_terms_freq': getters.get_artist_terms_freq(curFile), 'release': getters.get_release(curFile), 'song_id': getters.get_song_id(curFile), 'track_7digitalid': getters.get_track_7digitalid(curFile), 'title': getters.get_title(curFile), 'artist_latitude': getters.get_artist_latitude(curFile), 'energy': getters.get_energy(curFile), 'key': getters.get_key(curFile), 'release_7digitalid': getters.get_release_7digitalid(curFile), 'artist_mbid': getters.get_artist_mbid(curFile), 'segments_confidence': getters.get_segments_confidence(curFile), 'artist_hotttnesss': getters.get_artist_hotttnesss(curFile), 'time_signature': getters.get_time_signature(curFile), 'segments_loudness_max_time': getters.get_segments_loudness_max_time(curFile), 'mode': getters.get_mode(curFile), 'segments_loudness_start': getters.get_segments_loudness_start(curFile), 'tempo': getters.get_tempo(curFile), 'key_confidence': getters.get_key_confidence(curFile), 'analysis_sample_rate': getters.get_analysis_sample_rate(curFile), 'bars_confidence': getters.get_bars_confidence(curFile), 'artist_playmeid': getters.get_artist_playmeid(curFile), 'artist_terms_weight': getters.get_artist_terms_weight(curFile), 'segments_start': getters.get_segments_start(curFile), 'artist_location': getters.get_artist_location(curFile), 'loudness': getters.get_loudness(curFile), 'year': getters.get_year(curFile), 'artist_7digitalid': getters.get_artist_7digitalid(curFile), 'audio_md5': getters.get_audio_md5(curFile), 'segments_timbre': getters.get_segments_timbre(curFile), 'mode_confidence': getters.get_mode_confidence(curFile), 'end_of_fade_in': getters.get_end_of_fade_in(curFile), 'danceability': getters.get_danceability(curFile), 'artist_familiarity': getters.get_artist_familiarity(curFile), 'artist_mbtags_count': getters.get_artist_mbtags_count(curFile), 'tatums_start': getters.get_tatums_start(curFile), 'artist_id': getters.get_artist_id(curFile), 'segments_loudness_max': getters.get_segments_loudness_max(curFile), 'bars_start': getters.get_bars_start(curFile), 'beats_start': getters.get_beats_start(curFile), 'artist_terms': getters.get_artist_terms(curFile), 'sections_start': getters.get_sections_start(curFile), 'beats_confidence': getters.get_beats_confidence(curFile), 'sections_confidence': getters.get_sections_confidence(curFile)}
tid = fil.split('/')[-1].split('.')[0]
# print(c)
for i in c:
if i in get_table:
d2[i] = get_table[i]
d2[i] = str(d2[i]).replace('\n','')
build_str = build_str + d2[i] + ','
else:
print('error: unspecified field')
exit(0)
build_str = build_str[:-1]
# print(build_str[:-1])
build_str = build_str + '\n'
curFile.close()
build_str = build_str.replace('b','').replace("'",'').replace('"','')
return (build_str)
def get(getters, h5file):
# sanity check
if not os.path.isfile(h5file):
print 'ERROR: file', h5file, 'does not exist.'
sys.exit(0)
h5 = hdf5_getters.open_h5_file_read(h5file)
numSongs = hdf5_getters.get_num_songs(h5)
songidx = 0
if songidx >= numSongs:
print 'ERROR: file contains only',numSongs
h5.close()
sys.exit(0)
line = dict()
for getter in getters:
try:
res = hdf5_getters.__getattribute__('get_' + getter)(h5,songidx)
except AttributeError, e:
print e
if res.__class__.__name__ == 'ndarray':
# print getter[4:]+": shape =",res.shape
# How to put multidimensional values into file.
# Try to put only mean of the values etc...
print 'Ignoring....'
else:
# print getter[4:]+":",res
line[getter] = res
def extract_data(filename):
h5 = GETTERS.open_h5_file_read(filename)
#what data you want:
track_id = GETTERS.get_song_id(h5)
h5.close()
def get_btchromas_loudness(h5):
"""
Similar to btchroma, but adds the loudness back.
We use the segments_loudness_max
There is no max value constraint, simply no negative values.
"""
# if string, open and get chromas, if h5, get chromas
if type(h5).__name__ == "str":
h5 = GETTERS.open_h5_file_read(h5)
chromas = GETTERS.get_segments_pitches(h5)
segstarts = GETTERS.get_segments_start(h5)
btstarts = GETTERS.get_beats_start(h5)
duration = GETTERS.get_duration(h5)
loudnessmax = GETTERS.get_segments_loudness_max(h5)
h5.close()
else:
chromas = GETTERS.get_segments_pitches(h5)
segstarts = GETTERS.get_segments_start(h5)
btstarts = GETTERS.get_beats_start(h5)
duration = GETTERS.get_duration(h5)
loudnessmax = GETTERS.get_segments_loudness_max(h5)
# get the series of starts for segments and beats
segstarts = np.array(segstarts).flatten()
btstarts = np.array(btstarts).flatten()
# add back loudness
chromas = chromas.T * idB(loudnessmax)
# aligned features
btchroma = align_feats(chromas, segstarts, btstarts, duration)
if btchroma is None:
return None
# done (no renormalization)
return btchroma
def get_all_titles(basedir,targetdir,ext='.h5') :
global cap
decadecount = defaultdict(int)
i = 0
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
year = hdf5_getters.get_year(h5)
print i
i+=1
if year == 0:
h5.close()
continue
label = getbin(year)
if decadecount[label] > cap:
flag = checkforcompletion(decadecount)
h5.close()
if flag: # All the bins have exceeded their count. We can proceed with the training
return decadecount
continue
# Copy files
copy(f,targetdir)
decadecount[label] += 1
h5.close()
return decadecount
def func_to_extract_features(filename):
"""
This function extracts all features: per-track, per-section and per-segment
"""
# - open the song file
h5 = GETTERS.open_h5_file_read(filename)
# - get per-track features and put them
artist_id = GETTERS.get_artist_id(h5)
song_id = GETTERS.get_song_id(h5)
artist_familiarity = GETTERS.get_artist_familiarity(h5)
artist_hotttnesss = GETTERS.get_artist_hotttnesss(h5)
artist_latitude = GETTERS.get_artist_latitude(h5)
artist_longitude = GETTERS.get_artist_longitude(h5)
danceability = GETTERS.get_danceability(h5)
energy = GETTERS.get_energy(h5)
loudness = GETTERS.get_loudness(h5)
song_hotttnesss = GETTERS.get_song_hotttnesss(h5)
tempo = GETTERS.get_tempo(h5)
year = GETTERS.get_year(h5)
# artist_ids.add(artist_id)
# features_tuple = (artist_id, artist_familiarity, artist_hotttnesss, artist_latitude, artist_longitude, danceability, energy, loudness, song_hotttnesss, tempo, year)
features_tuple = (artist_id, artist_familiarity, artist_hotttnesss, loudness, song_hotttnesss, tempo, year)
# print features_tuple
features_tuples[song_id] = features_tuple
# files_per_artist[artist_id] += 1
# - close the file
h5.close()
def get_all_titles(basedir,ext='.h5') :
titles = []
artist_names = []
terms = []
loudness = []
segments_loudness_max = []
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
titles.append(hdf5_getters.get_title(h5))
artist_names.append(hdf5_getters.get_artist_name(h5))
try:
terms.append(hdf5_getters.get_artist_terms(h5))
except:
pass
loudness.append(hdf5_getters.get_loudness(h5))
try:
segments_loudness_max.append(hdf5_getters.get_segments_loudness_max(h5))
except:
pass
h5.close()
return titles, artist_names, terms, loudness, segments_loudness_max
def get_mode_advanced_feature(track,h5=None):
modern_modes = np.array([[1,0,1,0,1,1,0,1,0,1,0,1],
[1,0,1,1,0,1,0,1,0,1,1,0],
[1,1,0,1,0,1,0,1,1,0,1,0],
[1,0,1,0,1,0,1,1,0,1,0,1],
[1,0,1,0,1,1,0,1,0,1,1,0],
[1,0,1,1,0,1,0,1,1,0,1,0],
[1,1,0,1,0,1,1,0,1,0,1,0]])
close = (h5== None)
if h5 == None:
path = "../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5"
h5 = GETTERS.open_h5_file_read(path)
pitches = GETTERS.get_segments_pitches(h5)
maxpitches = pitches.argmax(axis=1)
important_pitches = np.bincount(maxpitches).argsort()[-7:] #get the 7 most important pitches
mode_advanced = np.zeros(12)
mode_advanced[important_pitches] = 1;
#key = GETTERS.get_key(h5)
#mode_advanced = np.roll(mode_advanced,-key)
#max = 11
#mode_advanced = 0
#for i in range(7):
# if(sum(mode_advanced != modern_modes[i,:])< max):
# max = sum(mode_advanced != modern_modes[i,:])
# mode_advanced = i
if close:
h5.close()
return mode_advanced
def get_bttimbre(h5):
"""
Get beat-aligned timbre from a song file of the Million Song Dataset
INPUT:
h5 - filename or open h5 file
RETURN:
bttimbre - beat-aligned timbre, one beat per column
or None if something went wrong (e.g. no beats)
"""
# if string, open and get timbre, if h5, get timbre
if type(h5).__name__ == "str":
h5 = GETTERS.open_h5_file_read(h5)
timbre = GETTERS.get_segments_timbre(h5)
segstarts = GETTERS.get_segments_start(h5)
btstarts = GETTERS.get_beats_start(h5)
duration = GETTERS.get_duration(h5)
h5.close()
else:
timbre = GETTERS.get_segments_timbre(h5)
segstarts = GETTERS.get_segments_start(h5)
btstarts = GETTERS.get_beats_start(h5)
duration = GETTERS.get_duration(h5)
# get the series of starts for segments and beats
# NOTE: MAYBE USELESS?
# result for track: 'TR0002Q11C3FA8332D'
# segstarts.shape = (708,)
# btstarts.shape = (304,)
segstarts = np.array(segstarts).flatten()
btstarts = np.array(btstarts).flatten()
# aligned features
bttimbre = align_feats(timbre.T, segstarts, btstarts, duration)
if bttimbre is None:
return None
# done (no renormalization)
return bttimbre
def csv_convert(basedir, csv_filename):
'''Function to convert all the information in a h5 file to a csv file, one song per line
Inputs: basedir, a string of subdirectory within the current directory
csv_filename, a filename where all the information will be written down'''
t1=time.time()
cnt=0
with open("/home/ec2-user/{}".format(csv_filename), "w") as csv_file:
csv_file.write(header)
for root, dirs, files in os.walk(basedir):
files=glob.glob(os.path.join(root, '*.h5'))
for f in files:
h5=gt.open_h5_file_read(f)
##each h5 file actually has multiple songs
num_songs=gt.get_num_songs(h5)
for j in range(int(num_songs)):
if validate_song(h5, j):
cnt+=1
csv_file.write(h5_to_csv_fields(h5,j))
#sanity check to make sure this is working
if cnt%10==0:
print("{} files csved thus far".format(cnt))
#remember to close your files or you run out of memory...
h5.close()
t2=time.time()
print ('directory {} csv-ed in:'.format(basedir), strtimedelta(t1,t2))
def main():
if len(sys.argv) != 2:
print ('Takes one argument, the directory with the data files.')
return
hdf5_files = get_all_files(sys.argv[1])
# Define properties to get
properties = ['danceability', 'duration', 'end_of_fade_in', 'energy', 'key', 'loudness', 'mode', 'song_hotttnesss', 'start_of_fade_out', 'tempo', 'time_signature', 'year', 'artist_terms']
count_datapoints = 0
with open('data.csv', 'w') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(properties)
for abspath in hdf5_files:
prop_values = []
h5 = hdf5_getters.open_h5_file_read(abspath)
hotness_check = hdf5_getters.get_song_hotttnesss(h5)
if math.isnan(hotness_check):
h5.close()
continue
for prop in properties:
prop_value = hdf5_getters.__getattribute__('get_' + prop)(h5)
#print (prop_value)
# special case artist terms to format it
if prop == 'artist_terms':
prop_values.append(';'.join(str(x) for x in prop_value))
else:
prop_values.append(str(prop_value))
csvwriter.writerow(prop_values)
count_datapoints += 1
h5.close();
print('Wrote % datapoints (lines)'.format(count_datapoints))
sys.exit(0)
def get_timbre(track, h5=None):
# returns
# 0 mean of all the timbre components in the song
# 1 variances of all timbre components in the song
# 2 median of all timbre components in the song
# 3 min component of all timbre components in the song
# 4 max component of all timbre components in the song
close = h5 == None
if h5 == None:
path = "../../msd_dense_subset/dense/" + track[2] + "/" + track[3] + "/" + track[4] + "/" + track + ".h5"
h5 = GETTERS.open_h5_file_read(path)
bttimbre = GETTERS.get_segments_timbre(h5)
# bttimbre = preprocessing.scale(bttimbre) #todo: ask if it's better to normalize here?
if close:
h5.close()
if bttimbre != None:
return (
bttimbre.mean(axis=0),
bttimbre.var(axis=0),
np.median(bttimbre, axis=0),
np.min(bttimbre, axis=0),
np.max(bttimbre, axis=0),
)
else:
return None
def traverseAndWrite(root, genreDirs, genreKeys):
if not isfile(root):
for f in listdir(root):
traverseAndWrite(root + "/" + f,genreDirs, genreKeys)
else:
h5 = hdf5_getters.open_h5_file_read(root)
numOfSongs = hdf5_getters.get_num_songs(h5)
for index in range(numOfSongs):
tags = hdf5_getters.get_artist_mbtags(h5,index)
# print tags
artist = hdf5_getters.get_artist_name(h5,index)
songName = hdf5_getters.get_title(h5,index)
segmentTimbre = hdf5_getters.get_segments_timbre(h5,index)
segmentPitches = hdf5_getters.get_segments_pitches(h5,index)
if notValidSong(tags, artist, songName, segmentTimbre, segmentPitches):
h5.close()
continue
for genre in genreKeys:
if genreInTags(genre,tags):
song = {}
song['genre'] = genre
song['artist_name'] = artist
song['song_title'] = songName
song['segments_pitches'] = segmentPitches.tolist()
song['segments_timbre'] = segmentTimbre.tolist()
valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits)
songName = ''.join(c for c in songName if c in valid_chars)
artist = ''.join(c for c in artist if c in valid_chars)
fd = open(genreDirs[genre]+"/"+artist+"--"+songName+".json",'a')
writeToDescriptor(fd,song)
fd.close()
h5.close()
def get_all_data(dir,ext,outDir):
songsDict = {}
i = 0
for root, dirs, files in os.walk(dir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = get.open_h5_file_read(f)
#songId = get.get_song_id(h5)
pitchAr = (np.array(get.get_segments_pitches(h5))).mean(axis=0)
timbreAr = (np.array(get.get_segments_timbre(h5))).mean(axis=0)
loudness = get.get_loudness(h5)
concatenatedArr = np.append([pitchAr,timbreAr],loudness)
featureSet[i] = concatenatedArr
#songsDict[songId] = featureSet[i]
print i
i += 1
h5.close()
#with open(outDir, "a") as f:
#for key in songsDict.keys():
#f.write("{0}|{1}".format(key,songsDict[key]))
#f.write("\n")
np.savetxt(outputDirPath,featureSet,delimiter = ',')
def feat_from_file(path):
"""
Extract a list of features in an array, already converted to string
"""
feats = []
h5 = GETTERS.open_h5_file_read(path)
# basic info
feats.append( GETTERS.get_track_id(h5) )
feats.append( GETTERS.get_artist_name(h5).replace(',','') )
feats.append( GETTERS.get_title(h5).replace(',','') )
feats.append( GETTERS.get_loudness(h5) )
feats.append( GETTERS.get_tempo(h5) )
feats.append( GETTERS.get_time_signature(h5) )
feats.append( GETTERS.get_key(h5) )
feats.append( GETTERS.get_mode(h5) )
feats.append( GETTERS.get_duration(h5) )
# timbre
timbre = GETTERS.get_segments_timbre(h5)
avg_timbre = np.average(timbre,axis=0)
for k in avg_timbre:
feats.append(k)
var_timbre = np.var(timbre,axis=0)
for k in var_timbre:
feats.append(k)
# done with h5 file
h5.close()
# makes sure we return strings
feats = map(lambda x: str(x), feats)
return feats
def getSongProperties(songCount = 3000, splitData = True):
songDict = {}
songIdDict = {}
songIdCount = 0
for root, dirs, files in os.walk(msd_subset_data_path):
files = glob.glob(os.path.join(root,'*.h5'))
for f in files:
h5 = GETTERS.open_h5_file_read(f)
tempo = GETTERS.get_tempo(h5)
danceability = GETTERS.get_danceability(h5)
energy = GETTERS.get_energy(h5)
loudness = GETTERS.get_loudness(h5)
#print GETTERS.get_artist_terms(h5)
timbre = GETTERS.get_segments_timbre(h5)
artist_hotness = GETTERS.get_artist_hotttnesss(h5)
song_key = GETTERS.get_key(h5)
songIdDict[GETTERS.get_song_id(h5)] = songIdCount
songDict[songIdCount] = [tempo,danceability,energy,loudness,artist_hotness,song_key]
songIdCount += 1
h5.close()
#if len(songDict) >2:
# break
#if len(songDict) >2:
# break
if songIdCount > songCount and splitData:
break
return songIdDict,songDict
def sanity_check_1thread(maindir=None,threadid=-1,nthreads=-1,allfiles=[]):
"""
Main function, check a bunch of files by opening every field in
getter.
"""
assert not maindir is None,'wrong param maindir'
assert threadid>-1,'wrong param threadid'
assert nthreads>0,'wrong param nthreads'
assert len(allfiles)>0,'wrong param allfiles, or no files'
# get getters
getters = filter(lambda x: x[:4] == 'get_', GETTERS.__dict__.keys())
# get the files to check
files_per_thread = int(np.ceil(len(allfiles) * 1. / nthreads))
p1 = files_per_thread * threadid
p2 = min(len(allfiles),files_per_thread * (threadid+1))
# iterate over files between p1 and p2
for f in allfiles[p1:p2]:
try:
h5 = GETTERS.open_h5_file_read(f)
for getter in getters:
tmp = GETTERS.__getattribute__(getter)(h5)
except KeyboardInterrupt:
raise KeyboardInterruptError()
except Exception,e:
print 'PROBLEM WITH FILE:',f; sys.stdout.flush()
raise
finally:
def parse_file(file, targetfile):
print "print parsing %s" %file
with open(file, 'r') as f:
global tracks, labels, features
tracks = list()
labels = list()
for line in f:
track, label = line.strip().split(' ')
tracks.append(track)
labels.append(int(label))
labels = np.array(labels)
features = np.empty((len(tracks),NUM_FEATURES),dtype='float')#hardcode number of features otherwise problem with scaling
track_info = np.empty((len(tracks)),dtype='object')
for i,track in enumerate(tracks):
#print track +' - ' +ut.get_track_info(track)
if(i%100 ==0):
print "processing track:%s\t%s" %(str(i),str(track))
if(file == SOURCE_DATA_FILE): #fetch h5 file from small dataset
h5 = GETTERS.open_h5_file_read("../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5") #fetch h5 file to allow faster preprocessing
else:
h5 = GETTERS.open_h5_file_read("../../msd_dense_subset/mood/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5") #fetch h5 file to allow faster preprocessing
track_info[i] = ut.get_track_info(track,h5)
timbre = ut.get_timbre(track,h5) #returns a tuple with 5 elements (12*5 = 60)
tempo = ut.get_tempo_feature(track,h5) #(1)
loudness = ut.get_loudness(track,h5)#returns a tuple with 3 elements (3)
energy = ut.get_energy_feature(track) #(1)
pitches = ut.get_pitches(track, h5) #(12)
features[i] = np.concatenate((timbre[0], timbre[1], timbre[2], timbre[3], timbre[4],pitches[0], pitches[1],pitches[2], pitches[3],pitches[4],np.array([tempo]),np.array(loudness),np.array([energy])))
h5.close()
print "done parsing"
print "saving data"
data = {
'features': features,
'labels': labels,
'tracks': tracks, #(songindex, songid)
'track_titles': track_info
}
with open(targetfile, 'w') as f:
pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
print "data saved to %s" % targetfile
#SCRIPT
#parse_file(SOURCE_DATA_FILE_2,E_TARGET_DATA_FILE_2)
#parse_file(SOURCE_DATA_FILE,E_TARGET_DATA_FILE)
def map(line): h5 = hdf5_getters.open_h5_file_read(line) if(h5): output_array=classifiers_base.classify(h5) yield(str(hdf5_getters.get_track_id(h5,0)),json.dumps(output_array)) h5.close() del output_array gc.collect()
def get_pitches(track,h5=None):
close = (h5== None)
if h5==None:
path = "../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5"
h5 = GETTERS.open_h5_file_read(path)
pitches = GETTERS.get_segments_pitches(h5)
if close:
h5.close()
return (pitches.mean(axis=0), pitches.var(axis=0), np.median(pitches,axis=0), np.min(pitches,axis=0), np.max(pitches,axis=0))
def song_to_artist(if_str):
songs_tracks = pickle.load(open ("../../msd_dense_subset/dense/songs_tracks.pkl",'r'));
track = str(songs_tracks[if_str])
# build path
path = "../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5"
h5 = GETTERS.open_h5_file_read(path)
artist = GETTERS.get_artist_name(h5)
h5.close()
return artist
def get_feature(self, track_id, feature):
if self.track_id != track_id:
if not self.h5 is None:
self.h5.close()
file_path = os.path.join(self,track_id[2],track_id[3],track_id[4],track_id+'.h5')
self.h5 = hdf5_getters.open_h5_file_read(file_path)
h5getter = getattr(hdf5_getters, 'get_'+feature)
return h5getter(self.h5,0)
def extractValues(hdf5path, summary, fields):
# summary = False
songidx = 0
onegetter = ''
# print hdf5path
h5 = hdf5_getters.open_h5_file_read(hdf5path)
# get all getters
keys = filter(lambda x: x[:4] == 'get_', hdf5_getters.__dict__.keys())
getters = []
keys.remove("get_num_songs") # special case
for onegetter in fields:
if onegetter[:4] != 'get_':
onegetter = 'get_' + onegetter #add get_
try:
keys.index(onegetter) #find if keyval exists else exit
except ValueError:
print 'ERROR: getter requested:',onegetter,'does not exist.'
h5.close()
sys.exit(0)
getters.append(onegetter)
getters = np.sort(getters)
retDict = {}
# print them
for getter in getters:
try:
res = hdf5_getters.__getattribute__(getter)(h5,songidx)
except AttributeError, e:
if summary:
continue
else:
print e
print 'forgot -summary flag? specified wrong getter?'
#print getter + "\n"
#print res
if res.__class__.__name__ == 'float64':
if math.isnan(res):
res = Global_Constant
if res.__class__.__name__ == 'ndarray':
#print getter[4:]+": shape =",res.shape
newlist = []
for i in res:
newlist.append(i)
#print newlist
retDict[getter[4:]] = newlist
else:
retDict[getter[4:]] = res
def get_all_titles(basedir,ext='.h5') :
titles = []
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
titles.append( hdf5_getters.get_title(h5) )
h5.close()
return titles
def get_time_signature(track,h5=None):
close = (h5== None)
if h5==None:
path = "../../msd_dense_subset/dense/"+track[2]+"/"+track[3]+"/"+track[4]+"/"+track+".h5"
h5 = GETTERS.open_h5_file_read(path)
time_signature = GETTERS.get_time_signature(h5)
if close:
h5.close()
return time_signature
def buildfeatures(basedir,cluster,ext='.h5'):
global cap
i = 0
features = []
decade = []
decadecount = defaultdict(int)
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root,'*'+ext))
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
year = hdf5_getters.get_year(h5)
if year == 0:
h5.close()
continue
bins = getbin(year)
# dec = (year/10)*10
# if dec < 1960:
# h5.close()
# continue
if decadecount[bins] > cap:
flag = checkforcompletion(decadecount)
h5.close()
if flag:
return features,decade
continue
i += 1
print i
clustercount = {}
for x in range(50):
clustercount[x] = 0
feature = []
try:
bttimbre = bt.get_bttimbre(h5)
btT = bttimbre.T
for x in btT:
label = kmeans.predict(x)
clustercount[label[0]] += 1
for cl in clustercount.keys():
feature.append(clustercount[cl])
features.append(feature)
decade.append(bins)
decadecount[bins] += 1
except:
h5.close()
continue
h5.close()
return features,decade
def process_filelist_train(filelist=None,
testartists=None,
tmpfilename=None,
npicks=None,
winsize=None,
finaldim=None,
typecompress='picks'):
"""
Main function, process all files in the list (as long as their artist
is not in testartist)
INPUT
filelist - a list of song files
testartists - set of artist ID that we should not use
tmpfilename - where to save our processed features
npicks - number of segments to pick per song
winsize - size of each segment we pick
finaldim - how many values do we keep
typecompress - one of 'picks' (win of btchroma), 'corrcoef' (correlation coefficients),
'cov' (covariance)
"""
# sanity check
for arg in locals().values():
assert not arg is None, 'process_filelist_train, missing an argument, something still None'
if os.path.isfile(tmpfilename):
print 'ERROR: file', tmpfilename, 'already exists.'
return
# create outputfile
output = tables.openFile(tmpfilename, mode='a')
group = output.createGroup("/", 'data', 'TMP FILE FOR YEAR RECOGNITION')
output.createEArray(group,
'feats',
tables.Float64Atom(shape=()), (0, finaldim),
'',
expectedrows=len(filelist))
output.createEArray(group,
'year',
tables.IntAtom(shape=()), (0, ),
'',
expectedrows=len(filelist))
output.createEArray(group,
'track_id',
tables.StringAtom(18, shape=()), (0, ),
'',
expectedrows=len(filelist))
# random projection
ndim = 12 # fixed in this dataset
if typecompress == 'picks':
randproj = RANDPROJ.proj_point5(ndim * winsize, finaldim)
elif typecompress == 'corrcoeff' or typecompress == 'cov':
randproj = RANDPROJ.proj_point5(ndim * ndim, finaldim)
elif typecompress == 'avgcov':
randproj = RANDPROJ.proj_point5(90, finaldim)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
# iterate over files
cnt_f = 0
for f in filelist:
cnt_f += 1
# verbose
if cnt_f % 50000 == 0:
print 'training... checking file #', cnt_f
# check file
h5 = GETTERS.open_h5_file_read(f)
artist_id = GETTERS.get_artist_id(h5)
year = GETTERS.get_year(h5)
track_id = GETTERS.get_track_id(h5)
h5.close()
if year <= 0 or artist_id in testartists:
continue
# we have a train artist with a song year, we're good
bttimbre = get_bttimbre(f)
if typecompress == 'picks':
if bttimbre is None:
continue
# we even have normal features, awesome!
processed_feats = CBTF.extract_and_compress(bttimbre,
npicks,
winsize,
finaldim,
randproj=randproj)
elif typecompress == 'corrcoeff':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.corr_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'cov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.cov_and_compress(timbres,
finaldim,
randproj=randproj)
elif typecompress == 'avgcov':
h5 = GETTERS.open_h5_file_read(f)
timbres = GETTERS.get_segments_timbre(h5).T
h5.close()
processed_feats = CBTF.avgcov_and_compress(timbres,
finaldim,
randproj=randproj)
else:
assert False, 'Unknown type of compression: ' + str(typecompress)
# save them to tmp file
n_p_feats = processed_feats.shape[0]
output.root.data.year.append(np.array([year] * n_p_feats))
output.root.data.track_id.append(np.array([track_id] * n_p_feats))
output.root.data.feats.append(processed_feats)
# we're done, close output
output.close()
return
# Test file to test the hdf5_getter
import hdf5_getters
# Open the song file
h5 = hdf5_getters.open_h5_file_read(
"../MillionSongSubset/data/A/A/A/TRAAAAW128F429D538.h5")
# Access the field "duration" in the file
duration = hdf5_getters.get_duration(h5)
# Print it
print duration
# Close the file
h5.close()
def get_all_rows(basedir, ext='.h5'):
rows = []
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
for f in files:
# print(os.path.join(root, f))
h5 = hdf5_getters.open_h5_file_read(f)
num_songs = hdf5_getters.get_num_songs(h5)
# print(num_songs)
for i in range(num_songs):
print(i)
obj = {}
obj['artist_name'] = hdf5_getters.get_artist_name(
h5, i).decode('UTF-8')
obj['artist_familiarity'] = hdf5_getters.get_artist_familiarity(
h5, i)
obj['artist_hotness'] = hdf5_getters.get_artist_hotttnesss(
h5, i)
obj['artist_id'] = hdf5_getters.get_artist_id(
h5, i).decode('UTF-8')
# obj['artist_mbid']=hdf5_getters.get_artist_mbid(h5,i).decode('UTF-8')
obj['artist_playmeid'] = hdf5_getters.get_artist_playmeid(
h5, i)
obj['artist_7digitalid'] = hdf5_getters.get_artist_7digitalid(
h5, i)
# obj['artist_latitude']=hdf5_getters.get_artist_latitude(h5,i)
# obj['artist_longitude']=hdf5_getters.get_artist_longitude(h5,i)
# obj['artist_location']=hdf5_getters.get_artist_location(h5,i).decode('UTF-8')
obj['artist_name'] = hdf5_getters.get_artist_name(
h5, i).decode('UTF-8')
obj['release'] = hdf5_getters.get_release(h5,
i).decode('UTF-8')
obj['song_hotttnesss'] = hdf5_getters.get_song_hotttnesss(
h5, i)
obj['title'] = hdf5_getters.get_title(h5, i).decode('UTF-8')
# obj['artist_terms']=hdf5_getters.get_artist_terms(h5)
# obj['artist_terms_freq']=hdf5_getters.get_artist_terms_freq(h5)
# obj['artist_terms_weight']=hdf5_getters.get_artist_terms_weight(h5)
# obj['audio_md5']=hdf5_getters.get_audio_md5(h5).decode('UTF-8')
obj['danceability'] = hdf5_getters.get_danceability(h5, i)
obj['duration'] = hdf5_getters.get_duration(h5, i)
obj['end_of_fade_in'] = hdf5_getters.get_end_of_fade_in(h5, i)
obj['energy'] = hdf5_getters.get_energy(h5, i)
obj['key'] = hdf5_getters.get_key(h5, i)
obj['key_confidence'] = hdf5_getters.get_key_confidence(h5, i)
obj['loudness'] = hdf5_getters.get_loudness(h5, i)
obj['mode'] = hdf5_getters.get_mode(h5, i)
# obj['start_of_fade_out']=hdf5_getters.get_start_of_fade_out(h5)
obj['tempo'] = hdf5_getters.get_tempo(h5, i)
obj['time_signature'] = hdf5_getters.get_time_signature(h5, i)
# obj['time_signature_confidence']=hdf5_getters.get_time_signature_confidence(h5)
obj['track_id'] = hdf5_getters.get_track_id(h5,
i).decode('UTF-8')
# obj['segments_start']=hdf5_getters.get_segments_start(h5)
# obj['segments_confidence']=hdf5_getters.get_segments_confidence(h5)
# obj['segments_pitches']=hdf5_getters.get_segments_pitches(h5)
# obj['segments_timbre']=hdf5_getters.get_segments_timbre(h5)
# obj['segments_loudness_max']=hdf5_getters.get_segments_loudness_max(h5)
# obj['segments_loudness_max_time']=hdf5_getters.get_segments_loudness_max_time(h5)
# obj['segments_confidence']=hdf5_getters.get_segments_confidence(h5)
# obj['segments_loudness_start']=hdf5_getters.get_segments_loudness_start(h5)
# obj['sections_start']=hdf5_getters.get_sections_start(h5)
# obj['sections_confidence']=hdf5_getters.get_sections_confidence(h5)
# obj['beats_start']=hdf5_getters.get_beats_start(h5)
# obj['beats_confidence']=hdf5_getters.get_beats_confidence(h5)
# obj['bars_start']=hdf5_getters.get_bars_start(h5)
# obj['bars_confidence']=hdf5_getters.get_bars_confidence(h5)
# obj['tatums_start']=hdf5_getters.get_tatums_start(h5)
# obj['artist_mbtags']=hdf5_getters.get_artist_mbtags(h5)
# obj['artist_mbtags_count']=hdf5_getters.get_artist_mbtags_count(h5)
obj['year'] = hdf5_getters.get_year(h5, i)
rows.append(obj)
h5.close()
return rows
import MySQLdb
import os
db = MySQLdb.connect(host="localhost",
user="******",
passwd="password",
db="FinalProject")
db.query("DELETE FROM artist WHERE artist_id = 'a';")
cursor = db.cursor(MySQLdb.cursors.DictCursor)
counter = 0
for subdir, dirs, files in os.walk("data/"):
for file in files:
f = os.path.join(subdir, file)
if ".h5" in f:
h5 = h.open_h5_file_read(f)
print("----------")
''' Store artist tuples '''
artist_id = h.get_artist_id(h5, 0)
artist_name = h.get_artist_name(h5, 0)
artist_name = artist_name.replace("'", "")
artist_hottness = str(h.get_artist_hotttnesss(h5, 0))
print artist_hottness
if artist_hottness == "nan":
artist_hottness = "0.0"
artist_familiarity = str(h.get_artist_familiarity(h5, 0))
if artist_familiarity == "nan":
artist_familiarity = "0.0"
cursor.execute("SELECT * FROM artist WHERE artist_id = '" +
artist_id + "'")
rs = cursor.fetchall()
def data_to_flat_file(basedir, ext='.h5'):
""" This function extracts the information from the tables and creates the flat file. """
count = 0
#song counter
list_to_write = []
group_index = 0
row_to_write = ""
writer = csv.writer(open("complete.csv", "wb"))
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
for f in files:
row = []
print f
h5 = hdf5_getters.open_h5_file_read(f)
title = hdf5_getters.get_title(h5)
title = title.replace('"', '')
row.append(title)
comma = title.find(',')
if comma != -1:
print title
time.sleep(1)
album = hdf5_getters.get_release(h5)
album = album.replace('"', '')
row.append(album)
comma = album.find(',')
if comma != -1:
print album
time.sleep(1)
artist_name = hdf5_getters.get_artist_name(h5)
comma = artist_name.find(',')
if comma != -1:
print artist_name
time.sleep(1)
artist_name = artist_name.replace('"', '')
row.append(artist_name)
duration = hdf5_getters.get_duration(h5)
row.append(duration)
samp_rt = hdf5_getters.get_analysis_sample_rate(h5)
row.append(samp_rt)
artist_7digitalid = hdf5_getters.get_artist_7digitalid(h5)
row.append(artist_7digitalid)
artist_fam = hdf5_getters.get_artist_familiarity(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(artist_fam) == True:
artist_fam = -1
row.append(artist_fam)
artist_hotness = hdf5_getters.get_artist_hotttnesss(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(artist_hotness) == True:
artist_hotness = -1
row.append(artist_hotness)
artist_id = hdf5_getters.get_artist_id(h5)
row.append(artist_id)
artist_lat = hdf5_getters.get_artist_latitude(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(artist_lat) == True:
artist_lat = -1
row.append(artist_lat)
artist_loc = hdf5_getters.get_artist_location(h5)
row.append(artist_loc)
artist_lon = hdf5_getters.get_artist_longitude(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(artist_lon) == True:
artist_lon = -1
row.append(artist_lon)
artist_mbid = hdf5_getters.get_artist_mbid(h5)
row.append(artist_mbid)
#Getting the genre
art_trm = hdf5_getters.get_artist_terms(h5)
trm_freq = hdf5_getters.get_artist_terms_freq(h5)
trn_wght = hdf5_getters.get_artist_terms_weight(h5)
a_mb_tags = hdf5_getters.get_artist_mbtags(h5)
genre_indexes = get_genre_indexes(
trm_freq) #index of the highest freq
genre_set = 0 #flag to see if the genre has been set or not
final_genre = []
genres_so_far = []
for i in range(len(genre_indexes)):
genre_tmp = get_genre(
art_trm, genre_indexes[i]
) #genre that corresponds to the highest freq
genres_so_far = genre_dict.get_genre_in_dict(
genre_tmp) #getting the genre from the dictionary
if len(genres_so_far) != 0:
for i in genres_so_far:
final_genre.append(i)
genre_set = 1
if genre_set == 1:
col_num = []
for i in final_genre:
column = int(i) #getting the column number of the genre
col_num.append(column)
genre_array = genre_columns(col_num) #genre array
for i in range(len(
genre_array)): #appending the genre_array to the row
row.append(genre_array[i])
else:
genre_array = genre_columns(
-1
) #when there is no genre matched, return an array of [0...0]
for i in range(len(
genre_array)): #appending the genre_array to the row
row.append(genre_array[i])
artist_pmid = hdf5_getters.get_artist_playmeid(h5)
row.append(artist_pmid)
audio_md5 = hdf5_getters.get_audio_md5(h5)
row.append(audio_md5)
danceability = hdf5_getters.get_danceability(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(danceability) == True:
danceability = -1
row.append(danceability)
end_fade_in = hdf5_getters.get_end_of_fade_in(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(end_fade_in) == True:
end_fade_in = -1
row.append(end_fade_in)
energy = hdf5_getters.get_energy(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(energy) == True:
energy = -1
row.append(energy)
song_key = hdf5_getters.get_key(h5)
row.append(song_key)
key_c = hdf5_getters.get_key_confidence(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(key_c) == True:
key_c = -1
row.append(key_c)
loudness = hdf5_getters.get_loudness(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(loudness) == True:
loudness = -1
row.append(loudness)
mode = hdf5_getters.get_mode(h5)
row.append(mode)
mode_conf = hdf5_getters.get_mode_confidence(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(mode_conf) == True:
mode_conf = -1
row.append(mode_conf)
release_7digitalid = hdf5_getters.get_release_7digitalid(h5)
row.append(release_7digitalid)
song_hot = hdf5_getters.get_song_hotttnesss(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(song_hot) == True:
song_hot = -1
row.append(song_hot)
song_id = hdf5_getters.get_song_id(h5)
row.append(song_id)
start_fade_out = hdf5_getters.get_start_of_fade_out(h5)
row.append(start_fade_out)
tempo = hdf5_getters.get_tempo(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(tempo) == True:
tempo = -1
row.append(tempo)
time_sig = hdf5_getters.get_time_signature(h5)
row.append(time_sig)
time_sig_c = hdf5_getters.get_time_signature_confidence(h5)
#checking if we get a "nan" if we do we change it to -1
if numpy.isnan(time_sig_c) == True:
time_sig_c = -1
row.append(time_sig_c)
track_id = hdf5_getters.get_track_id(h5)
row.append(track_id)
track_7digitalid = hdf5_getters.get_track_7digitalid(h5)
row.append(track_7digitalid)
year = hdf5_getters.get_year(h5)
row.append(year)
bars_c = hdf5_getters.get_bars_confidence(h5)
bars_start = hdf5_getters.get_bars_start(h5)
row_bars_padding = padding(
245
) #this is the array that will be attached at the end of th row
#--------------bars---------------"
gral_info = []
gral_info = row[:]
empty = []
for i, item in enumerate(bars_c):
row.append(group_index)
row.append(i)
row.append(bars_c[i])
bars_c_avg = get_avg(bars_c)
row.append(bars_c_avg)
bars_c_max = get_max(bars_c)
row.append(bars_c_max)
bars_c_min = get_min(bars_c)
row.append(bars_c_min)
bars_c_stddev = get_stddev(bars_c)
row.append(bars_c_stddev)
bars_c_count = get_count(bars_c)
row.append(bars_c_count)
bars_c_sum = get_sum(bars_c)
row.append(bars_c_sum)
row.append(bars_start[i])
bars_start_avg = get_avg(bars_start)
row.append(bars_start_avg)
bars_start_max = get_max(bars_start)
row.append(bars_start_max)
bars_start_min = get_min(bars_start)
row.append(bars_start_min)
bars_start_stddev = get_stddev(bars_start)
row.append(bars_start_stddev)
bars_start_count = get_count(bars_start)
row.append(bars_start_count)
bars_start_sum = get_sum(bars_start)
row.append(bars_start_sum)
for i in row_bars_padding:
row.append(i)
writer.writerow(row)
row = []
row = gral_info[:]
#--------beats---------------"
beats_c = hdf5_getters.get_beats_confidence(h5)
group_index = 1
row = []
row = gral_info[:]
row_front = padding(
14) #blanks left in front of the row(empty spaces for bars)
row_beats_padding = padding(231)
for i, item in enumerate(beats_c):
row.append(group_index)
row.append(i)
for index in row_front: #padding blanks in front of the beats
row.append(index)
row.append(beats_c[i])
beats_c_avg = get_avg(beats_c)
row.append(beats_c_avg)
beats_c_max = get_max(beats_c)
row.append(beats_c_max)
beats_c_min = get_min(beats_c)
row.append(beats_c_min)
beats_c_stddev = get_stddev(beats_c)
row.append(beats_c_stddev)
beats_c_count = get_count(beats_c)
row.append(beats_c_count)
beats_c_sum = get_sum(beats_c)
row.append(beats_c_sum)
beats_start = hdf5_getters.get_beats_start(h5)
row.append(beats_start[i])
beats_start_avg = get_avg(beats_start)
row.append(beats_start_avg)
beats_start_max = get_max(beats_start)
row.append(beats_start_max)
beats_start_min = get_min(beats_start)
row.append(beats_start_min)
beats_start_stddev = get_stddev(beats_start)
row.append(beats_start_stddev)
beats_start_count = get_count(beats_start)
row.append(beats_start_count)
beats_start_sum = get_sum(beats_start)
row.append(beats_start_sum)
for i in row_beats_padding:
row.append(i)
writer.writerow(row)
row = []
row = gral_info[:]
# "--------sections---------------"
row_sec_padding = padding(
217) #blank spaces left at the end of the row
sec_c = hdf5_getters.get_sections_confidence(h5)
group_index = 2
row = []
row = gral_info[:]
row_front = padding(
28) #blank spaces left in front(empty spaces for bars,beats)
for i, item in enumerate(sec_c):
row.append(group_index)
row.append(i)
for index in row_front: #padding blanks in front of the sections
row.append(index)
row.append(sec_c[i])
sec_c_avg = get_avg(sec_c)
row.append(sec_c_avg)
sec_c_max = get_max(sec_c)
row.append(sec_c_max)
sec_c_min = get_min(sec_c)
row.append(sec_c_min)
sec_c_stddev = get_stddev(sec_c)
row.append(sec_c_stddev)
sec_c_count = get_count(sec_c)
row.append(sec_c_count)
sec_c_sum = get_sum(sec_c)
row.append(sec_c_sum)
sec_start = hdf5_getters.get_sections_start(h5)
row.append(sec_start[i])
sec_start_avg = get_avg(sec_start)
row.append(sec_start_avg)
sec_start_max = get_max(sec_start)
row.append(sec_start_max)
sec_start_min = get_min(sec_start)
row.append(sec_start_min)
sec_start_stddev = get_stddev(sec_start)
row.append(sec_start_stddev)
sec_start_count = get_count(sec_start)
row.append(sec_start_count)
sec_start_sum = get_sum(sec_start)
row.append(sec_start_sum)
for i in row_sec_padding: #appending the blank spaces at the end of the row
row.append(i)
writer.writerow(row)
row = []
row = gral_info[:]
#--------segments---------------"
row_seg_padding = padding(182) #blank spaces at the end of the row
row_front = padding(42) #blank spaces left in front of segments
seg_c = hdf5_getters.get_segments_confidence(h5)
group_index = 3
row = []
row = gral_info[:]
for i, item in enumerate(seg_c):
row.append(group_index)
row.append(i)
for index in row_front: #padding blanks in front of the segments
row.append(index)
row.append(seg_c[i])
seg_c_avg = get_avg(seg_c)
row.append(seg_c_avg)
seg_c_max = get_max(seg_c)
row.append(seg_c_max)
seg_c_min = get_min(seg_c)
row.append(seg_c_min)
seg_c_stddev = get_stddev(seg_c)
row.append(seg_c_stddev)
seg_c_count = get_count(seg_c)
row.append(seg_c_count)
seg_c_sum = get_sum(seg_c)
row.append(seg_c_sum)
seg_loud_max = hdf5_getters.get_segments_loudness_max(h5)
row.append(seg_loud_max[i])
seg_loud_max_avg = get_avg(seg_loud_max)
row.append(seg_loud_max_avg)
seg_loud_max_max = get_max(seg_loud_max)
row.append(seg_loud_max_max)
seg_loud_max_min = get_min(seg_loud_max)
row.append(seg_loud_max_min)
seg_loud_max_stddev = get_stddev(seg_loud_max)
row.append(seg_loud_max_stddev)
seg_loud_max_count = get_count(seg_loud_max)
row.append(seg_loud_max_count)
seg_loud_max_sum = get_sum(seg_loud_max)
row.append(seg_loud_max_sum)
seg_loud_max_time = hdf5_getters.get_segments_loudness_max_time(
h5)
row.append(seg_loud_max_time[i])
seg_loud_max_time_avg = get_avg(seg_loud_max_time)
row.append(seg_loud_max_time_avg)
seg_loud_max_time_max = get_max(seg_loud_max_time)
row.append(seg_loud_max_time_max)
seg_loud_max_time_min = get_min(seg_loud_max_time)
row.append(seg_loud_max_time_min)
seg_loud_max_time_stddev = get_stddev(seg_loud_max_time)
row.append(seg_loud_max_time_stddev)
seg_loud_max_time_count = get_count(seg_loud_max_time)
row.append(seg_loud_max_time_count)
seg_loud_max_time_sum = get_sum(seg_loud_max_time)
row.append(seg_loud_max_time_sum)
seg_loud_start = hdf5_getters.get_segments_loudness_start(h5)
row.append(seg_loud_start[i])
seg_loud_start_avg = get_avg(seg_loud_start)
row.append(seg_loud_start_avg)
seg_loud_start_max = get_max(seg_loud_start)
row.append(seg_loud_start_max)
seg_loud_start_min = get_min(seg_loud_start)
row.append(seg_loud_start_min)
seg_loud_start_stddev = get_stddev(seg_loud_start)
row.append(seg_loud_start_stddev)
seg_loud_start_count = get_count(seg_loud_start)
row.append(seg_loud_start_count)
seg_loud_start_sum = get_sum(seg_loud_start)
row.append(seg_loud_start_sum)
seg_start = hdf5_getters.get_segments_start(h5)
row.append(seg_start[i])
seg_start_avg = get_avg(seg_start)
row.append(seg_start_avg)
seg_start_max = get_max(seg_start)
row.append(seg_start_max)
seg_start_min = get_min(seg_start)
row.append(seg_start_min)
seg_start_stddev = get_stddev(seg_start)
row.append(seg_start_stddev)
seg_start_count = get_count(seg_start)
row.append(seg_start_count)
seg_start_sum = get_sum(seg_start)
row.append(seg_start_sum)
for i in row_seg_padding: #appending blank spaces at the end of the row
row.append(i)
writer.writerow(row)
row = []
row = gral_info[:]
#----------segments pitch and timbre---------------"
row_seg2_padding = padding(
14) #blank spaces left at the end of the row
row_front = padding(
77) #blank spaces left at the front of the segments and timbre
seg_pitch = hdf5_getters.get_segments_pitches(h5)
transpose_pitch = seg_pitch.transpose(
) #this is to tranpose the matrix,so we can have 12 rows
group_index = 4
row = []
row = gral_info[:]
for i, item in enumerate(transpose_pitch[0]):
row.append(group_index)
row.append(i)
for index in row_front: #padding blanks in front of segments and timbre
row.append(index)
row.append(transpose_pitch[0][i])
seg_pitch_avg = get_avg(transpose_pitch[0])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[0])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[0])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[0])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[0])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[0])
row.append(seg_pitch_sum)
row.append(transpose_pitch[1][i])
seg_pitch_avg = get_avg(transpose_pitch[1])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[1])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[1])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[1])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[1])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[1])
row.append(seg_pitch_sum)
row.append(transpose_pitch[2][i])
seg_pitch_avg = get_avg(transpose_pitch[2])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[2])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[2])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[2])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[2])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[2])
row.append(seg_pitch_sum)
row.append(transpose_pitch[3][i])
seg_pitch_avg = get_avg(transpose_pitch[3])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[3])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[3])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[3])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[3])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[3])
row.append(seg_pitch_sum)
row.append(transpose_pitch[4][i])
seg_pitch_avg = get_avg(transpose_pitch[4])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[4])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[4])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[4])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[4])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[4])
row.append(seg_pitch_sum)
row.append(transpose_pitch[5][i])
seg_pitch_avg = get_avg(transpose_pitch[5])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[5])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[5])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[5])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[5])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[5])
row.append(seg_pitch_sum)
row.append(transpose_pitch[6][i])
seg_pitch_avg = get_avg(transpose_pitch[6])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[6])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[6])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[6])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[6])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[6])
row.append(seg_pitch_sum)
row.append(transpose_pitch[7][i])
seg_pitch_avg = get_avg(transpose_pitch[7])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[7])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[7])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[7])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[7])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[7])
row.append(seg_pitch_sum)
row.append(transpose_pitch[8][i])
seg_pitch_avg = get_avg(transpose_pitch[8])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[8])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[8])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[8])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[8])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[8])
row.append(seg_pitch_sum)
row.append(transpose_pitch[9][i])
seg_pitch_avg = get_avg(transpose_pitch[9])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[9])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[9])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[9])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[9])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[9])
row.append(seg_pitch_sum)
row.append(transpose_pitch[10][i])
seg_pitch_avg = get_avg(transpose_pitch[10])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[10])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[10])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[10])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[10])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[10])
row.append(seg_pitch_sum)
row.append(transpose_pitch[11][i])
seg_pitch_avg = get_avg(transpose_pitch[11])
row.append(seg_pitch_avg)
seg_pitch_max = get_max(transpose_pitch[11])
row.append(seg_pitch_max)
seg_pitch_min = get_min(transpose_pitch[11])
row.append(seg_pitch_min)
seg_pitch_stddev = get_stddev(transpose_pitch[11])
row.append(seg_pitch_stddev)
seg_pitch_count = get_count(transpose_pitch[11])
row.append(seg_pitch_count)
seg_pitch_sum = get_sum(transpose_pitch[11])
row.append(seg_pitch_sum)
#timbre arrays
seg_timbre = hdf5_getters.get_segments_timbre(h5)
transpose_timbre = seg_pitch.transpose(
) #tranposing matrix, to have 12 rows
row.append(transpose_timbre[0][i])
seg_timbre_avg = get_avg(transpose_timbre[0])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[0])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[0])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[0])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[0])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[0])
row.append(seg_timbre_sum)
row.append(transpose_timbre[1][i])
seg_timbre_avg = get_avg(transpose_timbre[1])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[1])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[1])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[1])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[1])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[1])
row.append(seg_timbre_sum)
row.append(transpose_timbre[2][i])
seg_timbre_avg = get_avg(transpose_timbre[2])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[2])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[2])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[2])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[2])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[2])
row.append(seg_timbre_sum)
row.append(transpose_timbre[3][i])
seg_timbre_avg = get_avg(transpose_timbre[3])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[3])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[3])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[3])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[3])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[3])
row.append(seg_timbre_sum)
row.append(transpose_timbre[4][i])
seg_timbre_avg = get_avg(transpose_timbre[4])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[4])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[4])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[4])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[4])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[4])
row.append(seg_timbre_sum)
row.append(transpose_timbre[5][i])
seg_timbre_avg = get_avg(transpose_timbre[5])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[5])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[5])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[5])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[5])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[5])
row.append(seg_timbre_sum)
row.append(transpose_timbre[6][i])
seg_timbre_avg = get_avg(transpose_timbre[6])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[6])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[6])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[6])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[6])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[6])
row.append(seg_timbre_sum)
row.append(transpose_timbre[7][i])
seg_timbre_avg = get_avg(transpose_timbre[7])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[7])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[7])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[7])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[7])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[7])
row.append(seg_timbre_sum)
row.append(transpose_timbre[8][i])
seg_timbre_avg = get_avg(transpose_timbre[8])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[8])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[8])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[8])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[8])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[8])
row.append(seg_timbre_sum)
row.append(transpose_timbre[9][i])
seg_timbre_avg = get_avg(transpose_timbre[9])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[9])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[9])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[9])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[9])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[9])
row.append(seg_timbre_sum)
row.append(transpose_timbre[10][i])
seg_timbre_avg = get_avg(transpose_timbre[10])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[10])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[10])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[10])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[10])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[10])
row.append(seg_timbre_sum)
row.append(transpose_timbre[11][i])
seg_timbre_avg = get_avg(transpose_timbre[11])
row.append(seg_timbre_avg)
seg_timbre_max = get_max(transpose_timbre[11])
row.append(seg_timbre_max)
seg_timbre_min = get_min(transpose_timbre[11])
row.append(seg_timbre_min)
seg_timbre_stddev = get_stddev(transpose_timbre[11])
row.append(seg_timbre_stddev)
seg_timbre_count = get_count(transpose_timbre[11])
row.append(seg_timbre_count)
seg_timbre_sum = get_sum(transpose_timbre[11])
row.append(seg_timbre_sum)
for item in row_seg2_padding:
row.append(item)
writer.writerow(row)
row = []
row = gral_info[:]
# "--------tatums---------------"
tatms_c = hdf5_getters.get_tatums_confidence(h5)
group_index = 5
row_front = padding(245) #blank spaces left in front of tatums
row = []
row = gral_info[:]
for i, item in enumerate(tatms_c):
row.append(group_index)
row.append(i)
for item in row_front: #appending blank spaces at the front of the row
row.append(item)
row.append(tatms_c[i])
tatms_c_avg = get_avg(tatms_c)
row.append(tatms_c_avg)
tatms_c_max = get_max(tatms_c)
row.append(tatms_c_max)
tatms_c_min = get_min(tatms_c)
row.append(tatms_c_min)
tatms_c_stddev = get_stddev(tatms_c)
row.append(tatms_c_stddev)
tatms_c_count = get_count(tatms_c)
row.append(tatms_c_count)
tatms_c_sum = get_sum(tatms_c)
row.append(tatms_c_sum)
tatms_start = hdf5_getters.get_tatums_start(h5)
row.append(tatms_start[i])
tatms_start_avg = get_avg(tatms_start)
row.append(tatms_start_avg)
tatms_start_max = get_max(tatms_start)
row.append(tatms_start_max)
tatms_start_min = get_min(tatms_start)
row.append(tatms_start_min)
tatms_start_stddev = get_stddev(tatms_start)
row.append(tatms_start_stddev)
tatms_start_count = get_count(tatms_start)
row.append(tatms_start_count)
tatms_start_sum = get_sum(tatms_start)
row.append(tatms_start_sum)
writer.writerow(row)
row = []
row = gral_info[:]
transpose_pitch = seg_pitch.transpose(
) #this is to tranpose the matrix,so we can have 12 rows
#arrays containing the aggregate values of the 12 rows
seg_pitch_avg = []
seg_pitch_max = []
seg_pitch_min = []
seg_pitch_stddev = []
seg_pitch_count = []
seg_pitch_sum = []
i = 0
#Getting the aggregate values in the pitches array
for row in transpose_pitch:
seg_pitch_avg.append(get_avg(row))
seg_pitch_max.append(get_max(row))
seg_pitch_min.append(get_min(row))
seg_pitch_stddev.append(get_stddev(row))
seg_pitch_count.append(get_count(row))
seg_pitch_sum.append(get_sum(row))
i = i + 1
#extracting information from the timbre array
transpose_timbre = seg_pitch.transpose(
) #tranposing matrix, to have 12 rows
#arrays containing the aggregate values of the 12 rows
seg_timbre_avg = []
seg_timbre_max = []
seg_timbre_min = []
seg_timbre_stddev = []
seg_timbre_count = []
seg_timbre_sum = []
i = 0
for row in transpose_timbre:
seg_timbre_avg.append(get_avg(row))
seg_timbre_max.append(get_max(row))
seg_timbre_min.append(get_min(row))
seg_timbre_stddev.append(get_stddev(row))
seg_timbre_count.append(get_count(row))
seg_timbre_sum.append(get_sum(row))
i = i + 1
h5.close()
count = count + 1
print count
'artist_terms_freq', 'artist_terms_weight', 'danceability',
'energy', 'key', 'mode', 'loudness', 'path', 'file', 'duration',
'artist_familiarity', 'similar_artists', 'artist_id', 'title',
'hotttnesss', 'year', 'latitude', 'longitude', 'midi_name'
])
i = 0
for parent_folder in os.listdir(path):
for sub_folder in os.listdir(path + '/' + parent_folder):
for child_folder in os.listdir(path + '/' + parent_folder +
'/' + sub_folder):
for file in os.listdir(path + '/' + parent_folder + '/' +
sub_folder + '/' + child_folder):
with h5.open_h5_file_read(path + '/' + parent_folder +
'/' + sub_folder + '/' +
child_folder + '/' +
file) as ds:
row = []
row += [h5.get_artist_terms(ds)]
row += [h5.get_artist_terms_freq(ds)]
row += [h5.get_artist_terms_weight(ds)]
row += [h5.get_danceability(ds)]
row += [h5.get_energy(ds)]
row += [h5.get_key(ds)]
row += [h5.get_mode(ds)]
row += [h5.get_loudness(ds)]
row += [
parent_folder + '/' + sub_folder + '/' +
child_folder + '/'
]
import hdf5_getters
h5 = hdf5_getters.open_h5_file_read("data/msd_summary_file.h5")
for i in range(hdf5_getters.get_num_songs(h5)):
print(hdf5_getters.get_song_id(h5, i))
h5.close()
def main():
outputFile1 = open('SongCSV.csv', 'w')
csvRowString = ""
#################################################
#if you want to prompt the user for the order of attributes in the csv,
#leave the prompt boolean set to True
#else, set 'prompt' to False and set the order of attributes in the 'else'
#clause
prompt = False
#################################################
if prompt == True:
while prompt:
prompt = False
csvAttributeString = raw_input(
"\n\nIn what order would you like the colums of the CSV file?\n"
+ "Please delineate with commas. The options are: " +
"AlbumName, AlbumID, ArtistID, ArtistLatitude, ArtistLocation, ArtistLongitude,"
+
" ArtistName, Danceability, Duration, KeySignature, KeySignatureConfidence, Tempo,"
+
" SongID, TimeSignature, TimeSignatureConfidence, Title, and Year.\n\n"
+
"For example, you may write \"Title, Tempo, Duration\"...\n\n"
+ "...or exit by typing 'exit'.\n\n")
csvAttributeList = re.split('\W+', csvAttributeString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'AlbumID'.lower():
csvRowString += 'AlbumID'
elif attribute == 'AlbumName'.lower():
csvRowString += 'AlbumName'
elif attribute == 'ArtistID'.lower():
csvRowString += 'ArtistID'
elif attribute == 'ArtistLatitude'.lower():
csvRowString += 'ArtistLatitude'
elif attribute == 'ArtistLocation'.lower():
csvRowString += 'ArtistLocation'
elif attribute == 'ArtistLongitude'.lower():
csvRowString += 'ArtistLongitude'
elif attribute == 'ArtistName'.lower():
csvRowString += 'ArtistName'
elif attribute == 'Danceability'.lower():
csvRowString += 'Danceability'
elif attribute == 'Duration'.lower():
csvRowString += 'Duration'
elif attribute == 'KeySignature'.lower():
csvRowString += 'KeySignature'
elif attribute == 'KeySignatureConfidence'.lower():
csvRowString += 'KeySignatureConfidence'
elif attribute == 'SongID'.lower():
csvRowString += "SongID"
elif attribute == 'Tempo'.lower():
csvRowString += 'Tempo'
elif attribute == 'TimeSignature'.lower():
csvRowString += 'TimeSignature'
elif attribute == 'TimeSignatureConfidence'.lower():
csvRowString += 'TimeSignatureConfidence'
elif attribute == 'Title'.lower():
csvRowString += 'Title'
elif attribute == 'Year'.lower():
csvRowString += 'Year'
elif attribute == 'Familiarity'.lower(): ####Added by us!
csvRowString += song.familiarity
elif attribute == 'artist_mbid'.lower():
csvRowString += song.artist_mbid
elif attribute == 'artist_playmeid'.lower():
csvRowString += song.artist_playmeid
elif attribute == 'artist_7digid'.lower():
csvRowString += song.artist_7digid
elif attribute == 'hottness'.lower():
csvRowString += song.hottness
elif attribute == 'song_hottness'.lower():
csvRowString += song.song_hottness
elif attribute == 'digitalid7'.lower():
csvRowString += song.digitalid7
elif attribute == 'similar_artists'.lower():
csvRowString += song.similar_artists
elif attribute == 'artist_terms'.lower():
csvRowString += song.artist_terms
elif attribute == 'art_terms_freq'.lower():
csvRowString += song.art_terms_freq
elif attribute == 'art_terms_weight'.lower():
csvRowString += song.art_terms_weight
elif attribute == 'a_sample_rate'.lower():
csvRowString += song.a_sample_rate
elif attribute == 'audio_md5'.lower():
csvRowString += song.audio_md5
elif attribute == 'end_of_fade_in'.lower():
csvRowString += song.end_of_fade_in
elif attribute == 'energy'.lower():
csvRowString += song.energy
elif attribute == 'loudness'.lower():
csvRowString += song.loudness
elif attribute == 'mode'.lower():
csvRowString += song.mode
elif attribute == 'mode_conf'.lower():
csvRowString += song.mode_conf
elif attribute == 'start_of_fade_out'.lower():
csvRowString += song.start_of_fade_out
elif attribute == 'trackid'.lower():
csvRowString += song.trackid
elif attribute == 'segm_start'.lower():
csvRowString += song.segm_start
elif attribute == 'segm_conf'.lower():
csvRowString += song.segm_conf
elif attribute == 'segm_pitch'.lower():
csvRowString += song.segm_pitch
elif attribute == 'segm_timbre'.lower():
csvRowString += song.segm_timbre
elif attribute == 'segm_max_loud'.lower():
csvRowString += song.segm_max_loud
elif attribute == 'segm_max_loud_time'.lower():
csvRowString += song.segm_max_loud_time
elif attribute == 'segm_loud_start'.lower():
csvRowString += song.segm_loud_start
elif attribute == 'sect_start'.lower():
csvRowString += song.sect_start
elif attribute == 'sect_conf'.lower():
csvRowString += song.sect_conf
elif attribute == 'beats_start'.lower():
csvRowString += song.beats_start
elif attribute == 'beats_conf'.lower():
csvRowString += song.beats_conf
elif attribute == 'bars_start'.lower():
csvRowString += song.bars_start
elif attribute == 'bars_conf'.lower():
csvRowString += song.bars_conf
elif attribute == 'tatums_start'.lower():
csvRowString += song.tatums_start
elif attribute == 'tatums_conf'.lower():
csvRowString += song.tatums_conf
elif attribute == 'artist_mbtags'.lower():
csvRowString += song.artist_mbtags
elif attribute == 'artist_mbtags_count'.lower():
csvRowString += song.artist_mbtags_count
elif attribute == 'Exit'.lower():
sys.exit()
else:
prompt = True
print("==============")
print("I believe there has been an error with the input.")
print("==============")
break
csvRowString += ","
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
#else, if you want to hard code the order of the csv file and not prompt
#the user,
else:
#################################################
#change the order of the csv file here
#Default is to list all available attributes (in alphabetical order)
csvRowString = "SongID,AlbumID,AlbumName,ArtistID,ArtistLatitude,ArtistLocation,ArtistLongitude,ArtistName,Danceability,Duration,KeySignature,KeySignatureConfidence,Tempo,TimeSignature,TimeSignatureConfidence,Title,Year,Familiarity,Artist_Mbid,Artist_PlaymeId,Artist_7didId,Hottness,Song_Hottness,7digitalid,A_Sample_Rate,Audio_Md5,End_Of_Fade_In,Energy,Loudness,Mode,Mode_Conf,Start_Of_Fade_Out,TrackId"
#################################################
csvAttributeList = re.split(',', csvRowString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
#################################################
#Set the basedir here, the root directory from which the search
#for files stored in a (hierarchical data structure) will originate
basedir = "/home/bigdata/smalltest/" # "." As the default means the current directory
ext = ".h5" #Set the extension here. H5 is the extension for HDF5 files.
#################################################
#FOR LOOP
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
for f in files:
print(f)
songH5File = hdf5_getters.open_h5_file_read(f)
song = Song(str(hdf5_getters.get_song_id(songH5File)))
# testDanceability = hdf5_getters.get_danceability(songH5File)
# print type(testDanceability)
# print ("Here is the danceability: ") + str(testDanceability)
song.artistID = str(hdf5_getters.get_artist_id(songH5File))
song.albumID = str(hdf5_getters.get_release_7digitalid(songH5File))
song.albumName = str(hdf5_getters.get_release(songH5File))
song.artistLatitude = str(
hdf5_getters.get_artist_latitude(songH5File))
song.artistLocation = str(
hdf5_getters.get_artist_location(songH5File))
song.artistLongitude = str(
hdf5_getters.get_artist_longitude(songH5File))
song.artistName = str(hdf5_getters.get_artist_name(songH5File))
song.danceability = str(hdf5_getters.get_danceability(songH5File))
song.duration = str(hdf5_getters.get_duration(songH5File))
# song.setGenreList()
song.keySignature = str(hdf5_getters.get_key(songH5File))
song.keySignatureConfidence = str(
hdf5_getters.get_key_confidence(songH5File))
# song.lyrics = None
# song.popularity = None
song.tempo = str(hdf5_getters.get_tempo(songH5File))
song.timeSignature = str(
hdf5_getters.get_time_signature(songH5File))
song.timeSignatureConfidence = str(
hdf5_getters.get_time_signature_confidence(songH5File))
song.title = str(hdf5_getters.get_title(songH5File))
song.year = str(hdf5_getters.get_year(songH5File))
#########Added by us!
song.familiarity = str(
hdf5_getters.get_artist_familiarity(songH5File))
song.artist_mbid = str(hdf5_getters.get_artist_mbid(songH5File))
song.artist_playmeid = str(
hdf5_getters.get_artist_playmeid(songH5File))
song.artist_7digid = str(
hdf5_getters.get_artist_7digitalid(songH5File))
song.hottness = str(hdf5_getters.get_artist_hotttnesss(songH5File))
song.song_hottness = str(
hdf5_getters.get_song_hotttnesss(songH5File))
song.digitalid7 = str(
hdf5_getters.get_track_7digitalid(songH5File))
#song.similar_artists = str(hdf5_getters.get_similar_artists(songH5File))
#song.artist_terms = str(hdf5_getters.get_artist_terms(songH5File))
#song.art_terms_freq = str(hdf5_getters.get_artist_terms_freq(songH5File))
#song.art_terms_weight = str(hdf5_getters.get_artist_terms_weight(songH5File))
song.a_sample_rate = str(
hdf5_getters.get_analysis_sample_rate(songH5File))
song.audio_md5 = str(hdf5_getters.get_audio_md5(songH5File))
song.end_of_fade_in = str(
hdf5_getters.get_end_of_fade_in(songH5File))
song.energy = str(hdf5_getters.get_energy(songH5File))
song.loudness = str(hdf5_getters.get_loudness(songH5File))
song.mode = str(hdf5_getters.get_mode(songH5File))
song.mode_conf = str(hdf5_getters.get_mode_confidence(songH5File))
song.start_of_fade_out = str(
hdf5_getters.get_start_of_fade_out(songH5File))
song.trackid = str(hdf5_getters.get_track_id(songH5File))
#song.segm_start = str(hdf5_getters.get_segments_start(songH5File))
#song.segm_conf = str(hdf5_getters.get_segments_confidence(songH5File))
#song.segm_pitch = str(hdf5_getters.get_segments_pitches(songH5File))
#song.segm_timbre = str(hdf5_getters.get_segments_timbre(songH5File))
#song.segm_max_loud = str(hdf5_getters.get_segments_loudness_max(songH5File))
#song.segm_max_loud_time = str(hdf5_getters.get_segments_loudness_max_time(songH5File))
#song.segm_loud_start = str(hdf5_getters.get_segments_loudness_start(songH5File))
#song.sect_start = str(hdf5_getters.get_sections_start(songH5File))
#song.sect_conf = str(hdf5_getters.get_sections_confidence(songH5File))
#song.beats_start = str(hdf5_getters.get_beats_start(songH5File))
#song.beats_conf = str(hdf5_getters.get_beats_confidence(songH5File))
#song.bars_start = str(hdf5_getters.get_bars_start(songH5File))
#song.bars_conf = str(hdf5_getters.get_bars_confidence(songH5File))
#song.tatums_start = str(hdf5_getters.get_tatums_start(songH5File))
#song.tatums_conf = str(hdf5_getters.get_tatums_confidence(songH5File))
#song.artist_mbtags = str(hdf5_getters.get_artist_mbtags(songH5File))
#song.artist_mbtags_count = str(hdf5_getters.get_artist_mbtags_count(songH5File))
#print song count
#csvRowString += str(song.songCount) + ","
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'AlbumID'.lower():
csvRowString += song.albumID
elif attribute == 'AlbumName'.lower():
albumName = song.albumName
albumName = albumName.replace("b\"", "")
albumName = albumName.replace("\"", "")
albumName = albumName.replace(',', "")
csvRowString += "\"" + albumName + "\""
elif attribute == 'ArtistID'.lower():
csvRowString += "\"" + song.artistID + "\""
elif attribute == 'ArtistLatitude'.lower():
latitude = song.artistLatitude
if latitude == 'nan':
latitude = ''
csvRowString += latitude
elif attribute == 'ArtistLocation'.lower():
location = song.artistLocation
location = location.replace(',', '')
location = location.replace("b\"", "")
location = location.replace("\"", "")
csvRowString += "\"" + location + "\""
elif attribute == 'ArtistLongitude'.lower():
longitude = song.artistLongitude
if longitude == 'nan':
longitude = ''
csvRowString += longitude
elif attribute == 'ArtistName'.lower():
artistName = song.artistName
artistName = artistName.replace("b\"", "")
artistName = artistName.replace("\"", "")
csvRowString += "\"" + artistName + "\""
elif attribute == 'Danceability'.lower():
csvRowString += song.danceability
elif attribute == 'Duration'.lower():
csvRowString += song.duration
elif attribute == 'KeySignature'.lower():
csvRowString += song.keySignature
elif attribute == 'KeySignatureConfidence'.lower():
# print "key sig conf: " + song.timeSignatureConfidence
csvRowString += song.keySignatureConfidence
elif attribute == 'SongID'.lower():
csvRowString += "\"" + song.id + "\""
elif attribute == 'Tempo'.lower():
# print "Tempo: " + song.tempo
csvRowString += song.tempo
elif attribute == 'TimeSignature'.lower():
csvRowString += song.timeSignature
elif attribute == 'TimeSignatureConfidence'.lower():
# print "time sig conf: " + song.timeSignatureConfidence
csvRowString += song.timeSignatureConfidence
elif attribute == 'Title'.lower():
t = song.title
t = t.replace("b\"", "")
t = t.replace("\"", "")
csvRowString += "\"" + t + "\""
elif attribute == 'Year'.lower():
csvRowString += song.year
elif attribute == 'Familiarity'.lower(): ####Added by us!
csvRowString += song.familiarity
elif attribute == 'artist_mbid'.lower():
csvRowString += "\"" + song.artist_mbid + "\""
elif attribute == 'artist_playmeid'.lower():
csvRowString += song.artist_playmeid
elif attribute == 'artist_7digid'.lower():
csvRowString += song.artist_7digid
elif attribute == 'hottness'.lower():
csvRowString += song.hottness
elif attribute == 'song_hottness'.lower():
csvRowString += song.song_hottness
elif attribute == 'digitalid7'.lower():
csvRowString += song.digitalid7
elif attribute == 'similar_artists'.lower():
csvRowString += song.similar_artists
elif attribute == 'artist_terms'.lower():
csvRowString += song.artist_terms
elif attribute == 'art_terms_freq'.lower():
csvRowString += song.art_terms_freq
elif attribute == 'art_terms_weight'.lower():
csvRowString += song.art_terms_weight
elif attribute == 'a_sample_rate'.lower():
csvRowString += song.a_sample_rate
elif attribute == 'audio_md5'.lower():
csvRowString += "\"" + song.audio_md5 + "\""
elif attribute == 'end_of_fade_in'.lower():
csvRowString += song.end_of_fade_in
elif attribute == 'energy'.lower():
csvRowString += song.energy
elif attribute == 'loudness'.lower():
csvRowString += song.loudness
elif attribute == 'mode'.lower():
csvRowString += song.mode
elif attribute == 'mode_conf'.lower():
csvRowString += song.mode_conf
elif attribute == 'start_of_fade_out'.lower():
csvRowString += song.start_of_fade_out
elif attribute == 'trackid'.lower():
csvRowString += "\"" + song.trackid + "\""
elif attribute == 'segm_start'.lower():
csvRowString += song.segm_start
elif attribute == 'segm_conf'.lower():
csvRowString += song.segm_conf
elif attribute == 'segm_pitch'.lower():
csvRowString += song.segm_pitch
elif attribute == 'segm_timbre'.lower():
csvRowString += song.segm_timbre
elif attribute == 'segm_max_loud'.lower():
csvRowString += song.segm_max_loud
elif attribute == 'segm_max_loud_time'.lower():
csvRowString += song.segm_max_loud_time
elif attribute == 'segm_loud_start'.lower():
csvRowString += song.segm_loud_start
elif attribute == 'sect_start'.lower():
csvRowString += song.sect_start
elif attribute == 'sect_conf'.lower():
csvRowString += song.sect_conf
elif attribute == 'beats_start'.lower():
csvRowString += song.beats_start
elif attribute == 'beats_conf'.lower():
csvRowString += song.beats_conf
elif attribute == 'bars_start'.lower():
csvRowString += song.bars_start
elif attribute == 'bars_conf'.lower():
csvRowString += song.bars_conf
elif attribute == 'tatums_start'.lower():
csvRowString += song.tatums_start
elif attribute == 'tatums_conf'.lower():
csvRowString += song.tatums_conf
elif attribute == 'artist_mbtags'.lower():
csvRowString += song.artist_mbtags
elif attribute == 'artist_mbtags_count'.lower():
csvRowString += song.artist_mbtags_count
else:
csvRowString += "\"ERR\""
csvRowString += ","
#Remove the final comma from each row in the csv
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
songH5File.close()
outputFile1.close()
while not os.path.exists(os.path.join(aligned_path, '{}.{}.pdf'.format(base, n))):
n += 1
return '{}.{}.mid'.format(base, n)
def to_h5_path(filename):
''' Given an mp3 filename, returns the path to the corresponding -beats.npy file '''
return os.path.join(base_data_path, get_data_folder(filename), 'msd', filename.replace('.mp3', '.h5'))
def to_midi_path(filename):
''' Given an mp3 filename, returns the path to the corresponding midi file '''
return os.path.join(base_data_path, get_data_folder(filename), midi_directory, to_numbered_mid(filename))
# Load in list of files which were aligned correctly, and the start/end times of the good alignment
files, start_times, end_times = load_results(tsv_path)
for filename, start_time, end_time in zip(files, start_times, end_times):
# Load in MSD hdf5 file
h5 = hdf5_getters.open_h5_file_read(to_h5_path(filename))
# Load in beat times from MSD
beats = hdf5_getters.get_beats_start(h5)
# Some files have no EN analysis
if beats.size == 0:
continue
# Get indices which fall within the range of correct alignment
time_mask = np.logical_and(beats > start_time, beats < end_time)
beats = beats[time_mask]
# and beat-synchronous feature matrices, within the time range of correct alignment
chroma = beat_aligned_feats.get_btchromas(h5)[:, time_mask]
timbre = beat_aligned_feats.get_bttimbre(h5)[:, time_mask]
loudness = beat_aligned_feats.get_btloudnessmax(h5)[:, time_mask]
h5.close()
# Stack it
msd_features = np.vstack([chroma, timbre, loudness])
import glob
import hdf5_getters
import csv
from tqdm import tqdm
myfile = open('info.csv', 'wb')
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
glob_path = '/Users/andrew/Documents/datamining/Project/MillionSongSubset/data/*/*/*/*'
filepaths = glob.glob(glob_path)
for filepath in tqdm(filepaths):
h5 = hdf5_getters.open_h5_file_read(filepath)
n = hdf5_getters.get_num_songs(h5)
# print n
for row in range(n):
artist = hdf5_getters.get_artist_name(h5,songidx=row)
song_id = hdf5_getters.get_song_id(h5,songidx=row)
title= hdf5_getters.get_title(h5,songidx=row)
info = [song_id, title, artist]
wr.writerow(info)
h5.close()
import hdf5_getters
h5 = hdf5_getters.open_h5_file_read(path to some file)
duration = hdf5_getters.get_duration(h5)
h5.close()
def get_timbre(file_name):
h5 = hdf5_getters.open_h5_file_read(file_name)
timbre = hdf5_getters.get_segments_timbre(h5)
h5.close()
return [timbre[:100].flatten()]
def main():
outputFile1 = open('SongCSV.csv', 'w')
csvRowString = ""
#################################################
#if you want to prompt the user for the order of attributes in the csv,
#leave the prompt boolean set to True
#else, set 'prompt' to False and set the order of attributes in the 'else'
#clause
prompt = False
#################################################
if prompt == True:
while prompt:
prompt = False
csvAttributeString = raw_input(
"\n\nIn what order would you like the colums of the CSV file?\n"
+ "Please delineate with commas. The options are: " +
"AlbumName, AlbumID, ArtistID, ArtistLocation," +
" ArtistName, Hotness, Duration, GenreList, KeySignature, KeySignatureConfidence, Tempo,"
+ " Loudness" +
" TimeSignature, TimeSignatureConfidence, Title, and Year.\n\n"
+
"For example, you may write \"Title, Tempo, Duration\"...\n\n"
+ "...or exit by typing 'exit'.\n\n")
csvAttributeList = re.split('\W+', csvAttributeString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'AlbumID'.lower():
csvRowString += 'AlbumID'
elif attribute == 'AlbumName'.lower():
csvRowString += 'AlbumName'
elif attribute == 'ArtistID'.lower():
csvRowString += 'ArtistID'
elif attribute == 'ArtistLocation'.lower():
csvRowString += 'ArtistLocation'
elif attribute == 'ArtistName'.lower():
csvRowString += 'ArtistName'
elif attribute == 'Hotness'.lower():
csvRowString += 'Hotness'
elif attribute == 'Duration'.lower():
csvRowString += 'Duration'
elif attribute == 'KeySignature'.lower():
csvRowString += 'KeySignature'
elif attribute == 'KeySignatureConfidence'.lower():
csvRowString += 'KeySignatureConfidence'
elif attribute == 'SongID'.lower():
csvRowString += "SongID"
elif attribute == 'Tempo'.lower():
csvRowString += 'Tempo'
elif attribute == 'TimeSignature'.lower():
csvRowString += 'TimeSignature'
elif attribute == 'TimeSignatureConfidence'.lower():
csvRowString += 'TimeSignatureConfidence'
elif attribute == 'Title'.lower():
csvRowString += 'Title'
elif attribute == 'Year'.lower():
csvRowString += 'Year'
elif attribute == 'Loudness'.lower():
csvRowString += 'Loudness'
elif attribute == 'Exit'.lower():
sys.exit()
else:
prompt = True
print("==============")
print("I believe there has been an error with the input.")
print("==============")
break
csvRowString += ","
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
#else, if you want to hard code the order of the csv file and not prompt
#the user,
else:
#################################################
#change the order of the csv file here
#Default is to list all available attributes (in alphabetical order)
csvRowString = (
"SongID,AlbumID,AlbumName,ArtistID,ArtistLocation," +
"ArtistName,Hotness,Duration,KeySignature," +
"KeySignatureConfidence,Tempo,TimeSignature,TimeSignatureConfidence,"
+ "Title,Year,Loudness")
#################################################
csvAttributeList = re.split('\W+', csvRowString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
outputFile1.write("SongNumber,")
outputFile1.write(csvRowString + "\n")
csvRowString = ""
#################################################
#Set the basedir here, the root directory from which the search
#for files stored in a (hierarchical data structure) will originate
basedir = "." # "." As the default means the current directory
ext = ".h5" #Set the extension here. H5 is the extension for HDF5 files.
#################################################
#FOR LOOP
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
for f in files:
print(f)
songH5File = hdf5_getters.open_h5_file_read(f)
song = Song(str(hdf5_getters.get_song_id(songH5File)))
testDanceability = hdf5_getters.get_danceability(songH5File)
# print type(testDanceability)
# print ("Here is the danceability: ") + str(testDanceability)
song.albumID = str(hdf5_getters.get_release_7digitalid(songH5File))
song.albumName = str(hdf5_getters.get_release(songH5File))
song.artistID = str(hdf5_getters.get_artist_id(songH5File))
song.artistLocation = str(
hdf5_getters.get_artist_location(songH5File))
song.artistName = str(hdf5_getters.get_artist_name(songH5File))
song.hotness = str(hdf5_getters.get_artist_hotttnesss(songH5File))
song.duration = str(hdf5_getters.get_duration(songH5File))
# song.setGenreList()
song.keySignature = str(hdf5_getters.get_key(songH5File))
song.keySignatureConfidence = str(
hdf5_getters.get_key_confidence(songH5File))
# song.lyrics = None
# song.popularity = None
song.tempo = str(hdf5_getters.get_tempo(songH5File))
song.timeSignature = str(
hdf5_getters.get_time_signature(songH5File))
song.timeSignatureConfidence = str(
hdf5_getters.get_time_signature_confidence(songH5File))
song.title = str(hdf5_getters.get_title(songH5File))
song.year = str(hdf5_getters.get_year(songH5File))
song.loudness = str(hdf5_getters.get_loudness(songH5File))
#print song count
csvRowString += str(song.songCount) + ","
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'SongID'.lower():
csvRowString += "\"" + song.id + "\""
elif attribute == 'AlbumID'.lower():
csvRowString += song.albumID
elif attribute == 'AlbumName'.lower():
albumName = song.albumName
albumName = albumName.replace(',', "")
csvRowString += "\"" + albumName + "\""
elif attribute == 'ArtistID'.lower():
csvRowString += "\"" + song.artistID + "\""
elif attribute == 'ArtistLocation'.lower():
location = song.artistLocation
location = location.replace(',', '')
csvRowString += "\"" + location + "\""
elif attribute == 'ArtistName'.lower():
csvRowString += "\"" + song.artistName + "\""
elif attribute == 'Hotness'.lower():
csvRowString += song.hotness
elif attribute == 'Duration'.lower():
csvRowString += song.duration
elif attribute == 'KeySignature'.lower():
csvRowString += song.keySignature
elif attribute == 'KeySignatureConfidence'.lower():
# print "key sig conf: " + song.timeSignatureConfidence
csvRowString += song.keySignatureConfidence
elif attribute == 'Tempo'.lower():
# print "Tempo: " + song.tempo
csvRowString += song.tempo
elif attribute == 'TimeSignature'.lower():
csvRowString += song.timeSignature
elif attribute == 'TimeSignatureConfidence'.lower():
# print "time sig conf: " + song.timeSignatureConfidence
csvRowString += song.timeSignatureConfidence
elif attribute == 'Title'.lower():
csvRowString += "\"" + song.title + "\""
elif attribute == 'Year'.lower():
csvRowString += song.year
elif attribute == 'Loudness'.lower():
csvRowString += "\"" + song.loudness + "\""
else:
csvRowString += "Erm. This didn't work. Error. :( :(\n"
csvRowString += ","
#Remove the final comma from each row in the csv
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
songH5File.close()
outputFile1.close()
def main():
basedir = "./../songMetaInfo.txt"
ext = ".h5"
if len(sys.argv) > 1:
basedir = sys.argv[1]
outputfile = 'SongFileMetaData.csv'
if len(sys.argv) > 2:
outputfile = sys.argv[2]
csvWriter = open(outputfile, 'w')
csvWriter.write(
"title,songId,artistId,artistfamilarity,artistHotness,songHotness," +
"songEnfOfFadeIn,startFadeout,energy,loudness,albumID,albumName,artistName,danceability,duration,keySignatureConfidence,tempo,timeSignature,timeSignatureConfidence,year\n"
)
with open(basedir) as file:
for line in file.readlines():
f = line.strip()
#newf = f + "text"
print f
#print f
try:
songH5File = hdf5_getters.open_h5_file_read(f)
csvStr = ""
#0
title = str(hdf5_getters.get_title(songH5File))
csvStr += title + ","
#1
songId = str(hdf5_getters.get_song_id(songH5File))
csvStr += songId + ","
#2
artistId = str(hdf5_getters.get_artist_id(songH5File))
csvStr += artistId + ","
#3
artistfamilarity = str(
hdf5_getters.get_artist_familiarity(songH5File))
csvStr += artistfamilarity + ","
#4
artistHotness = str(
hdf5_getters.get_artist_hotttnesss(songH5File))
csvStr += artistHotness + ","
#5
songHotness = str(hdf5_getters.get_song_hotttnesss(songH5File))
csvStr += songHotness + ","
#6
songEnfOfFadeIn = str(
hdf5_getters.get_end_of_fade_in(songH5File))
csvStr += songEnfOfFadeIn + ","
#7
startFadeOut = str(
hdf5_getters.get_start_of_fade_out(songH5File))
csvStr += startFadeOut + ","
#8
energy = str(hdf5_getters.get_energy(songH5File))
csvStr += energy + ","
#9
loudness = str(hdf5_getters.get_loudness(songH5File))
csvStr += loudness + ","
#10
albumID = str(hdf5_getters.get_release_7digitalid(songH5File))
csvStr += albumID + ","
#11
albumName = str(hdf5_getters.get_release(songH5File))
csvStr += albumName + ","
#12
artistName = str(hdf5_getters.get_artist_name(songH5File))
csvStr += artistName + ","
#13
danceability = str(hdf5_getters.get_danceability(songH5File))
csvStr += danceability + ","
#14
duration = str(hdf5_getters.get_duration(songH5File))
csvStr += duration + ","
#15
keySignatureConfidence = str(
hdf5_getters.get_key_confidence(songH5File))
csvStr += keySignatureConfidence + ","
#16
tempo = str(hdf5_getters.get_tempo(songH5File))
csvStr += tempo + ","
## 17
timeSignature = str(
hdf5_getters.get_time_signature(songH5File))
csvStr += timeSignature + ","
#18
timeSignatureConfidence = str(
hdf5_getters.get_time_signature_confidence(songH5File))
csvStr += timeSignatureConfidence + ","
#19
year = str(hdf5_getters.get_year(songH5File))
csvStr += year + ","
#print song count
csvStr += "\n"
csvWriter.write(csvStr)
#print csvStr
songH5File.close()
except:
print "Error in processing file"
csvWriter.close()
def get_year_bpm(file):
h5 = hdf5_getters.open_h5_file_read(file)
bpm = hdf5_getters.get_tempo(h5)
year = hdf5_getters.get_year(h5)
return {bpm: bpm, year: year}
# Column headers
# Currently only 4 features being extracted
# We can add as many as we want, just seperate with commas
csvfile.write("artist_name,title,artist_location,release,hotttness,familiarity,danceability,duration,energy,loudness,year,tempo,analysis_rate,end_of_fade_in,key,key_confidence,mode,mode_confidence,start_of_fade_out,time_signature,time_signature_conf,track_id")
#csvfile.write("familiarity, hotttness")
csvfile.write("\n")
# Recursively visit each sub-dir till we reach the h5 files
# Strip punctuation from features that are strings
for root, dirs, filenames in os.walk(indir):
for f in filenames:
#log = open(os.path.join(root, f),'r')
#print os.path.join(root, f)
# Use the hd5 wrappers to open the h5 file
h5 = hdf5_getters.open_h5_file_read(os.path.join(root, f))
# EXTRACTING FEATURES
# See hd5_getter.py for the various features that we can extract from each h5 file
# Get the artist name
artist_name = hdf5_getters.get_artist_name(h5)
artist = artist_name.translate(None, string.punctuation)
# Get the title of the song
title_song = hdf5_getters.get_title(h5)
title = title_song.translate(None, string.punctuation)
# Get artist location
artist_location = hdf5_getters.get_artist_location(h5)
def main():
basedir = "D:/Master K"
ext = ".H5" # Set the extension here. H5 is the extension for HDF5 files.
songs = []
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
#songs = {}
#keys = list()
#values = list()
for f in files:
print(f)
songH5File = hdf5_getters.open_h5_file_read(f)
song = Song(str(hdf5_getters.get_song_id(songH5File)))
item = {"song_id": song.id.replace('b', '')}
song.artistID = str(hdf5_getters.get_artist_id(songH5File))
song.artistID = song.artistID.replace('b', '', 1)
item["song_artistID"] = song.artistID
song.albumID = str(hdf5_getters.get_release_7digitalid(songH5File))
song.albumID = song.albumID.replace('b', '', 1)
item["song_albumID"] = song.albumID
song.albumName = str(hdf5_getters.get_release(songH5File))
song.albumName = song.albumName.replace('b', '', 1)
item["song_albumName"] = song.albumName
song.artistLatitude = str(
hdf5_getters.get_artist_latitude(songH5File))
song.artistLatitude = song.artistLatitude.replace('b', '', 1)
item["song_artistLatitude"] = song.artistLatitude
song.artistLocation = str(
hdf5_getters.get_artist_location(songH5File))
song.artistLocation = song.artistLocation.replace('b', '', 1)
item["song_artistLocation"] = song.artistLocation
song.artistLongitude = str(
hdf5_getters.get_artist_longitude(songH5File))
song.artistLongitude = song.artistLongitude.replace('b', '', 1)
item["song_artistLongitude"] = song.artistLongitude
song.artistName = str(hdf5_getters.get_artist_name(songH5File))
song.artistName = song.artistName.replace('b', '', 1)
item["song_artistName"] = song.artistName
song.danceability = str(hdf5_getters.get_danceability(songH5File))
song.danceability = song.danceability.replace('b', '', 1)
item["song_danceability"] = song.danceability
song.duration = str(hdf5_getters.get_duration(songH5File))
song.duration = song.duration.replace('b', '', 1)
item["song_duration"] = song.duration
song.keySignature = str(hdf5_getters.get_key(songH5File))
song.keySignature = song.keySignature.replace('b', '', 1)
item["song_keySignature"] = song.keySignature
song.keySignatureConfidence = str(
hdf5_getters.get_key_confidence(songH5File))
song.keySignatureConfidence = song.keySignatureConfidence.replace(
'b', '', 1)
item["song_keySignatureConfidence"] = song.keySignatureConfidence
song.tempo = str(hdf5_getters.get_tempo(songH5File))
song.tempo = song.tempo.replace('b', '', 1)
item["song_tempo"] = song.tempo
song.timeSignature = str(
hdf5_getters.get_time_signature(songH5File))
song.timeSignature = song.timeSignature.replace('b', '', 1)
item["song_timeSignature"] = song.timeSignature
song.timeSignatureConfidence = str(
hdf5_getters.get_time_signature_confidence(songH5File))
song.timeSignatureConfidence = song.timeSignatureConfidence.replace(
'b', '', 1)
item["song_timeSignatureConfidence"] = song.timeSignatureConfidence
song.title = str(hdf5_getters.get_title(songH5File))
song.title = song.title.replace('b', '', 1)
item["song_title"] = song.title
song.year = str(hdf5_getters.get_year(songH5File))
song.year = song.year.replace('b', '', 1)
item["song_year"] = song.year
#song.mfcc = str(hdf5_getters.get_segments_timbre(songH5File))
#item["song_mfcc"] = song.mfcc
item["song_mfcc"] = list(
hdf5_getters.get_segments_timbre(songH5File))
song.hotness = str(hdf5_getters.get_artist_hotttnesss(songH5File))
item["song_hotness"] = song.hotness
songs.append(item)
songH5File.close()
#song_dict= dict(zip(keys, values))
with open("D:\data_file_k.json", "w") as write_file:
json.dump(songs, write_file, cls=NumpyEncoder)
def hd5_single_random_file_parser():
# Open an h5 file in read mode
h5 = hdf5_getters.open_h5_file_read(
'/home/skalogerakis/Documents/MillionSong/MillionSongSubset/A/M/G/TRAMGDX12903CEF79F.h5'
)
function_tracker = filter(
lambda x: x.startswith('get'),
hdf5_getters.__dict__.keys()) # Detects all the getter functions
for f in function_tracker: # Print everything in function tracker
print(f)
# First effort to check what each field contains.
print() # 55 available fields (exluding number of songs fields)
print("Num of songs -- ",
hdf5_getters.get_num_songs(h5)) # One song per file
print("Title -- ",
hdf5_getters.get_title(h5)) # Print the title of a specific h5 file
print("Artist familiarity -- ", hdf5_getters.get_artist_familiarity(h5))
print("Artist hotness -- ", hdf5_getters.get_artist_hotttnesss(h5))
print("Artist ID -- ", hdf5_getters.get_artist_id(h5))
print("Artist mbID -- ", hdf5_getters.get_artist_mbid(h5))
print("Artist playmeid -- ", hdf5_getters.get_artist_playmeid(h5))
print("Artist 7DigitalID -- ", hdf5_getters.get_artist_7digitalid(h5))
print("Artist latitude -- ", hdf5_getters.get_artist_latitude(h5))
print("Artist longitude -- ", hdf5_getters.get_artist_longitude(h5))
print("Artist location -- ", hdf5_getters.get_artist_location(h5))
print("Artist Name -- ", hdf5_getters.get_artist_name(h5))
print("Release -- ", hdf5_getters.get_release(h5))
print("Release 7DigitalID -- ", hdf5_getters.get_release_7digitalid(h5))
print("Song ID -- ", hdf5_getters.get_song_id(h5))
print("Song Hotness -- ", hdf5_getters.get_song_hotttnesss(h5))
print("Track 7Digital -- ", hdf5_getters.get_track_7digitalid(h5))
print("Similar artists -- ", hdf5_getters.get_similar_artists(h5))
print("Artist terms -- ", hdf5_getters.get_artist_terms(h5))
print("Artist terms freq -- ", hdf5_getters.get_artist_terms_freq(h5))
print("Artist terms weight -- ", hdf5_getters.get_artist_terms_weight(h5))
print("Analysis sample rate -- ",
hdf5_getters.get_analysis_sample_rate(h5))
print("Audio md5 -- ", hdf5_getters.get_audio_md5(h5))
print("Danceability -- ", hdf5_getters.get_danceability(h5))
print("Duration -- ", hdf5_getters.get_duration(h5))
print("End of Fade -- ", hdf5_getters.get_end_of_fade_in(h5))
print("Energy -- ", hdf5_getters.get_energy(h5))
print("Key -- ", hdf5_getters.get_key(h5))
print("Key Confidence -- ", hdf5_getters.get_key_confidence(h5))
print("Loudness -- ", hdf5_getters.get_loudness(h5))
print("Mode -- ", hdf5_getters.get_mode(h5))
print("Mode Confidence -- ", hdf5_getters.get_mode_confidence(h5))
print("Start of fade out -- ", hdf5_getters.get_start_of_fade_out(h5))
print("Tempo -- ", hdf5_getters.get_tempo(h5))
print("Time signature -- ", hdf5_getters.get_time_signature(h5))
print("Time signature confidence -- ",
hdf5_getters.get_time_signature_confidence(h5))
print("Track ID -- ", hdf5_getters.get_track_id(h5))
print("Segments Start -- ", hdf5_getters.get_segments_start(h5))
print("Segments Confidence -- ", hdf5_getters.get_segments_confidence(h5))
print("Segments Pitches -- ", hdf5_getters.get_segments_pitches(h5))
print("Segments Timbre -- ", hdf5_getters.get_segments_timbre(h5))
print("Segments Loudness max -- ",
hdf5_getters.get_segments_loudness_max(h5))
print("Segments Loudness max time-- ",
hdf5_getters.get_segments_loudness_max_time(h5))
print("Segments Loudness start -- ",
hdf5_getters.get_segments_loudness_start(h5))
print("Sections start -- ", hdf5_getters.get_sections_start(h5))
print("Sections Confidence -- ", hdf5_getters.get_sections_confidence(h5))
print("Beats start -- ", hdf5_getters.get_beats_start(h5))
print("Beats confidence -- ", hdf5_getters.get_beats_confidence(h5))
print("Bars start -- ", hdf5_getters.get_bars_start(h5))
print("Bars confidence -- ", hdf5_getters.get_bars_confidence(h5))
print("Tatums start -- ", hdf5_getters.get_tatums_start(h5))
print("Tatums confidence -- ", hdf5_getters.get_tatums_confidence(h5))
print("Artist mbtags -- ", hdf5_getters.get_artist_mbtags(h5))
print("Artist mbtags count -- ", hdf5_getters.get_artist_mbtags_count(h5))
print("Year -- ", hdf5_getters.get_year(h5))
fields = ['Title', 'Artist ID']
with open('Tester2.csv', 'w', newline='') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=';')
# writing the fields
csv_writer.writerow(fields)
# writing the data rows
csv_writer.writerow(
[hdf5_getters.get_title(h5),
hdf5_getters.get_artist_id(h5)])
h5.close() # close h5 when completed in the end
def preprocess(filename, all_data, i):
data, all_artist_names, all_artist_ids, all_artist_locations, all_titles, all_song_ids, all_years, all_durations, all_modes, all_tempos, all_artist_mbtags = all_data
# data = [[]]
j = 0
h5 = GETTERS.open_h5_file_read(filename)
artist_name = GETTERS.get_artist_name(h5).decode("utf-8")
value = get_key_value(all_artist_names, artist_name)
data[i, j] = value
j += 1
artist_id = GETTERS.get_artist_id(h5).decode("utf-8")
value = get_key_value(all_artist_ids, artist_id)
data[i, j] = value
j += 1
artist_location = GETTERS.get_artist_location(h5).decode("utf-8")
value = get_key_value(all_artist_locations, artist_location)
data[i, j] = value
j += 1
title = GETTERS.get_title(h5).decode("utf-8")
value = get_key_value(all_titles, title)
data[i, j] = value
j += 1
song_id = GETTERS.get_song_id(h5).decode("utf-8")
value = get_key_value(all_song_ids, song_id)
data[i, j] = value
j += 1
year = GETTERS.get_year(h5)
all_years.add(year)
data[i, j] = year
j += 1
duration = GETTERS.get_duration(h5)
all_durations.add(duration)
data[i, j] = duration
j += 1
mode = GETTERS.get_mode(h5)
all_modes.add(mode)
data[i, j] = mode
j += 1
tempo = GETTERS.get_tempo(h5)
all_tempos.add(tempo)
data[i, j] = tempo
j += 1
artist_mbtags = GETTERS.get_artist_mbtags(h5)
k = 0
for artist_mbtag in artist_mbtags:
if k <= 4:
value = get_key_value(all_artist_mbtags,
artist_mbtag.decode("utf-8"))
data[i, j] = value
j += 1
k += 1
h5.close()
def complete_hd5_to_csv(basedir):
ext = '.h5' # Get all files with extension .h5
# Header title. Essentially it is a schema for all the following songs
header = [
'Title', 'Artist familiarity', 'Artist hotness', 'Artist ID',
'Artist mbID', 'Artist playmeid', 'Artist 7DigitalID',
'Artist latitude', 'Artist longitude', 'Artist location',
'Artist Name', 'Release', 'Release 7DigitalID', 'Song ID',
'Song Hotness', 'Track 7Digital', 'Analysis sample rate', 'Audio md5',
'Danceability', 'Duration', 'End of Fade', 'Energy', 'Key',
'Key Confidence', 'Loudness', 'Mode', 'Mode Confidence',
'Start of fade out', 'Tempo', 'Time signature',
'Time signature confidence', 'Track ID', 'Year'
]
with open('Tester2.csv', 'w', newline='') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=';')
# writing the header line. This line contains the schema of the data
csv_writer.writerow(header)
# Read all files from the given directories
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, '*' + ext))
print(files)
for f in files:
h5 = hdf5_getters.open_h5_file_read(f)
# Write as row all elements. NOTE: Only the serialized elements are parsed and not arrays
csv_writer.writerow([
hdf5_getters.get_title(h5),
hdf5_getters.get_artist_familiarity(h5),
hdf5_getters.get_artist_hotttnesss(h5),
hdf5_getters.get_artist_id(h5),
hdf5_getters.get_artist_mbid(h5),
hdf5_getters.get_artist_playmeid(h5),
hdf5_getters.get_artist_7digitalid(h5),
hdf5_getters.get_artist_latitude(h5),
hdf5_getters.get_artist_longitude(h5),
hdf5_getters.get_artist_location(h5),
hdf5_getters.get_artist_name(h5),
hdf5_getters.get_release(h5),
hdf5_getters.get_release_7digitalid(h5),
hdf5_getters.get_song_id(h5),
hdf5_getters.get_song_hotttnesss(h5),
hdf5_getters.get_track_7digitalid(h5),
hdf5_getters.get_analysis_sample_rate(h5),
hdf5_getters.get_audio_md5(h5),
hdf5_getters.get_danceability(h5),
hdf5_getters.get_duration(h5),
hdf5_getters.get_end_of_fade_in(h5),
hdf5_getters.get_energy(h5),
hdf5_getters.get_key(h5),
hdf5_getters.get_key_confidence(h5),
hdf5_getters.get_loudness(h5),
hdf5_getters.get_mode(h5),
hdf5_getters.get_mode_confidence(h5),
hdf5_getters.get_start_of_fade_out(h5),
hdf5_getters.get_tempo(h5),
hdf5_getters.get_time_signature(h5),
hdf5_getters.get_time_signature_confidence(h5),
hdf5_getters.get_track_id(h5),
hdf5_getters.get_year(h5)
])
# For debugging purposes. Everything as expected
# print()
# print("Num of songs -- ", hdf5_getters.get_num_songs(h5)) # One song per file
# print("Title -- ", hdf5_getters.get_title(h5)) # Print the title of a specific h5 file
# print("Artist familiarity -- ", hdf5_getters.get_artist_familiarity(h5))
# print("Artist hotness -- ", hdf5_getters.get_artist_hotttnesss(h5))
# print("Artist ID -- ", hdf5_getters.get_artist_id(h5))
# print("Artist mbID -- ", hdf5_getters.get_artist_mbid(h5))
# print("Artist playmeid -- ", hdf5_getters.get_artist_playmeid(h5))
# print("Artist 7DigitalID -- ", hdf5_getters.get_artist_7digitalid(h5))
# print("Artist latitude -- ", hdf5_getters.get_artist_latitude(h5))
# print("Artist longitude -- ", hdf5_getters.get_artist_longitude(h5))
# print("Artist location -- ", hdf5_getters.get_artist_location(h5))
# print("Artist Name -- ", hdf5_getters.get_artist_name(h5))
# print("Release -- ", hdf5_getters.get_release(h5))
# print("Release 7DigitalID -- ", hdf5_getters.get_release_7digitalid(h5))
# print("Song ID -- ", hdf5_getters.get_song_id(h5))
# print("Song Hotness -- ", hdf5_getters.get_song_hotttnesss(h5))
# print("Track 7Digital -- ", hdf5_getters.get_track_7digitalid(h5))
# print("Analysis sample rate -- ", hdf5_getters.get_analysis_sample_rate(h5))
# print("Audio md5 -- ", hdf5_getters.get_audio_md5(h5))
# print("Danceability -- ", hdf5_getters.get_danceability(h5))
# print("Duration -- ", hdf5_getters.get_duration(h5))
# print("End of Fade -- ", hdf5_getters.get_end_of_fade_in(h5))
# print("Energy -- ", hdf5_getters.get_energy(h5))
# print("Key -- ", hdf5_getters.get_key(h5))
# print("Key Confidence -- ", hdf5_getters.get_key_confidence(h5))
# print("Loudness -- ", hdf5_getters.get_loudness(h5))
# print("Mode -- ", hdf5_getters.get_mode(h5))
# print("Mode Confidence -- ", hdf5_getters.get_mode_confidence(h5))
# print("Start of fade out -- ", hdf5_getters.get_start_of_fade_out(h5))
# print("Tempo -- ", hdf5_getters.get_tempo(h5))
# print("Time signature -- ", hdf5_getters.get_time_signature(h5))
# print("Time signature confidence -- ", hdf5_getters.get_time_signature_confidence(h5))
# print("Track ID -- ", hdf5_getters.get_track_id(h5))
# # print("Artist mbtags -- ", hdf5_getters.get_artist_mbtags(h5))
# # print("Artist mbtags count -- ", hdf5_getters.get_artist_mbtags_count(h5))
# print("Year -- ", hdf5_getters.get_year(h5))
h5.close()
#from matplotlib.mlab import PCA
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D
import hdf5_getters as get
import numpy as np
#./A/A/A/TRAAAVO128F93133D4.h5
h5 = get.open_h5_file_read('../data/MillionSongSubset/data/B/I/I/TRBIIAL128F4252C04.h5')
segStart = get.get_segments_start(h5)
pitch = np.array(get.get_segments_pitches(h5))
pitch_flatten = pitch.flatten()
#timbrePCA = PCA(timbre)
#timbrePCA.fracs
#print timbrePCA.Y
print pitch.shape
print len(segStart)
print len(pitch_flatten)
#print get.get_song_id(h5)
"""
timbre = get.get_segments_timbre(h5,0)
timbre.shape()
print timbre[:1]"""
def main():
outputFile1 = open('SongCSV.csv', 'w')
csvRowString = ""
#################################################
#if you want to prompt the user for the order of attributes in the csv,
#leave the prompt boolean set to True
#else, set 'prompt' to False and set the order of attributes in the 'else'
#clause
prompt = False
#################################################
if prompt == True:
while prompt:
prompt = False
csvAttributeString = raw_input(
"\n\nIn what order would you like the colums of the CSV file?\n"
+ "Please delineate with commas. The options are: " +
"AlbumName, AlbumID, ArtistID, ArtistLatitude, ArtistLocation, ArtistLongitude,"
+
" ArtistName, Danceability, Duration, KeySignature, KeySignatureConfidence, Tempo,"
+
" SongID, TimeSignature, TimeSignatureConfidence, Title, Year and Hotttnesss.\n\n"
+
"For example, you may write \"Title, Tempo, Duration\"...\n\n"
+ "...or exit by typing 'exit'.\n\n")
csvAttributeList = re.split('\W+', csvAttributeString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'AlbumID'.lower():
csvRowString += 'AlbumID'
elif attribute == 'AlbumName'.lower():
csvRowString += 'AlbumName'
elif attribute == 'ArtistID'.lower():
csvRowString += 'ArtistID'
elif attribute == 'ArtistLatitude'.lower():
csvRowString += 'ArtistLatitude'
elif attribute == 'ArtistLocation'.lower():
csvRowString += 'ArtistLocation'
elif attribute == 'ArtistLongitude'.lower():
csvRowString += 'ArtistLongitude'
elif attribute == 'ArtistName'.lower():
csvRowString += 'ArtistName'
elif attribute == 'Danceability'.lower():
csvRowString += 'Danceability'
elif attribute == 'Duration'.lower():
csvRowString += 'Duration'
elif attribute == 'KeySignature'.lower():
csvRowString += 'KeySignature'
elif attribute == 'KeySignatureConfidence'.lower():
csvRowString += 'KeySignatureConfidence'
elif attribute == 'SongID'.lower():
csvRowString += "SongID"
elif attribute == 'Tempo'.lower():
csvRowString += 'Tempo'
elif attribute == 'TimeSignature'.lower():
csvRowString += 'TimeSignature'
elif attribute == 'TimeSignatureConfidence'.lower():
csvRowString += 'TimeSignatureConfidence'
elif attribute == 'Title'.lower():
csvRowString += 'Title'
elif attribute == 'Year'.lower():
csvRowString += 'Year'
elif attribute == 'SongHotttnesss'.lower():
csvRowString += 'SongHotttnesss'
elif attribute == 'Exit'.lower():
sys.exit()
else:
prompt = True
print("==============")
print("I believe there has been an error with the input.")
print("==============")
break
csvRowString += ","
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
#else, if you want to hard code the order of the csv file and not prompt
#the user,
else:
#################################################
#change the order of the csv file here
#Default is to list all available attributes (in alphabetical order)
csvRowString = (
"SongID,AlbumID,AlbumName,ArtistFamiliarity,ArtistHotttnesss,ArtistID,"
+ "ArtistLatitude,ArtistLocation," +
"ArtistLongitude,ArtistName,BarsConfidence,BarsStart,BeatsConfidence,"
+
"BeatsStart,Danceability,Duration,EndOfFadeIn,Energy,KeySignature,"
+
"KeySignatureConfidence,Loudness,Mode,ModeConfidence,SectionsConfidence,"
+ "SectionsStart,SegmentsConfidence,SegmentsLoudnessMax," +
"SegmentsLoudnessMaxTime,SegmentsLoudnessMaxStart,SegmentsPitches,"
+ "SegmentsStart,SegmentsTimbre,SongHotttnesss,TatumsConfidence," +
"TatumsStart,Tempo,TimeSignature," +
"TimeSignatureConfidence,Title,Year")
'''
csvRowString = ("SongID,AlbumID,AlbumName,ArtistFamiliarity,"+
"ArtistHotttnesss,ArtistID,"+
"ArtistLatitude,ArtistLocation,"+
"ArtistLongitude,ArtistName,"+
"BarsConfidence,BarsStart,"+
"Danceability,Duration,EndOfFadeIn,Energy,KeySignature,"+
"KeySignatureConfidence,Loudness,Mode,ModeConfidence,"+
"SegmentsPitches,"+
"SegmentsTimbre,SongHotttnesss,"+
"Tempo,TimeSignature,"+
"TimeSignatureConfidence,Title,Year")
'''
#################################################
csvAttributeList = re.split('\W+', csvRowString)
for i, v in enumerate(csvAttributeList):
csvAttributeList[i] = csvAttributeList[i].lower()
outputFile1.write("SongNumber,")
outputFile1.write(csvRowString + "\n")
csvRowString = ""
#################################################
#Set the basedir here, the root directory from which the search
#for files stored in a (hierarchical data structure) will originate
#basedir = "./I/"
basedir = "./MillionSongSubset/data/" # "." As the default means the current directory
#basedir = "/Users/dafirebanks/Downloads/MillionSongSubset/data/" # "." As the default means the current directory
ext = "*.h5" #Set the extension here. H5 is the extension for HDF5 files.
#################################################
#FOR LOOP
for root, dirs, files in os.walk(basedir):
files = glob.glob(os.path.join(root, ext))
for f in files:
print(f)
songH5File = hdf5_getters.open_h5_file_read(f)
song = Song(str(hdf5_getters.get_song_id(songH5File)))
#testDanceability = hdf5_getters.get_danceability(songH5File)
# print type(testDanceability)
# print ("Here is the danceability: ") + str(testDanceability)
song.artistFamiliarity = str(
hdf5_getters.get_artist_familiarity(songH5File))
song.artistHotttnesss = str(
hdf5_getters.get_artist_hotttnesss(songH5File))
song.artistID = str(hdf5_getters.get_artist_id(songH5File))
song.albumID = str(hdf5_getters.get_release_7digitalid(songH5File))
song.albumName = str(hdf5_getters.get_release(songH5File))
song.artistLatitude = str(
hdf5_getters.get_artist_latitude(songH5File))
song.artistLocation = str(
hdf5_getters.get_artist_location(songH5File))
song.artistLongitude = str(
hdf5_getters.get_artist_longitude(songH5File))
song.artistName = str(hdf5_getters.get_artist_name(songH5File))
song.barsConfidence = str(
hdf5_getters.get_bars_confidence(songH5File)).replace(
",", "").replace("\n", "")
song.barsStart = str(
hdf5_getters.get_bars_start(songH5File)).replace(",",
"").replace(
"\n", "")
song.beatsConfidence = str(
hdf5_getters.get_beats_confidence(songH5File)).replace(
",", "").replace("\n", "")
song.beatsStart = str(
hdf5_getters.get_beats_start(songH5File)).replace(",",
"").replace(
"\n", "")
song.danceability = str(hdf5_getters.get_danceability(songH5File))
song.duration = str(hdf5_getters.get_duration(songH5File))
song.endOfFadeIn = str(hdf5_getters.get_end_of_fade_in(songH5File))
song.energy = str(hdf5_getters.get_energy(songH5File))
# song.setGenreList()
song.hotttnesss = str(hdf5_getters.get_song_hotttnesss(songH5File))
song.keySignature = str(hdf5_getters.get_key(songH5File))
song.keySignatureConfidence = str(
hdf5_getters.get_key_confidence(songH5File))
song.loudness = str(hdf5_getters.get_loudness(songH5File))
song.mode = str(hdf5_getters.get_mode(songH5File))
song.modeConfidence = str(
hdf5_getters.get_mode_confidence(songH5File))
# song.lyrics = None
# song.popularity = None
song.sectionsConfidence = str(
hdf5_getters.get_sections_confidence(songH5File)).replace(
",", "").replace("\n", "")
song.sectionsStart = str(
hdf5_getters.get_segments_start(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsConfidence = str(
hdf5_getters.get_segments_confidence(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsLoudnessMax = str(
hdf5_getters.get_segments_loudness_max(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsLoudnessMaxTime = str(
hdf5_getters.get_segments_loudness_max_time(
songH5File)).replace(",", "").replace("\n", "")
song.segmentsLoudnessMaxStart = str(
hdf5_getters.get_segments_loudness_start(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsPitches = str(
hdf5_getters.get_segments_pitches(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsStart = str(
hdf5_getters.get_segments_start(songH5File)).replace(
",", "").replace("\n", "")
song.segmentsTimbre = str(
hdf5_getters.get_segments_timbre(songH5File)).replace(
",", "").replace("\n", "")
song.startOfFadeOut = str(
hdf5_getters.get_start_of_fade_out(songH5File))
song.tatumsConfidence = str(
hdf5_getters.get_tatums_confidence(songH5File)).replace(
",", "").replace("\n", "")
song.tatumsStart = str(
hdf5_getters.get_tatums_start(songH5File)).replace(
",", "").replace("\n", "")
song.tempo = str(hdf5_getters.get_tempo(songH5File))
song.timeSignature = str(
hdf5_getters.get_time_signature(songH5File))
song.timeSignatureConfidence = str(
hdf5_getters.get_time_signature_confidence(songH5File))
song.title = str(hdf5_getters.get_title(songH5File))
song.year = str(hdf5_getters.get_year(songH5File))
#print song count
csvRowString += str(song.songCount) + ","
for attribute in csvAttributeList:
# print "Here is the attribute: " + attribute + " \n"
if attribute == 'AlbumID'.lower():
csvRowString += song.albumID
elif attribute == 'AlbumName'.lower():
albumName = song.albumName
albumName = albumName.replace(',', "")
csvRowString += "\"" + albumName + "\""
elif attribute == 'ArtistFamiliarity'.lower():
csvRowString += song.artistFamiliarity
elif attribute == 'ArtistHotttnesss'.lower():
csvRowString += song.artistHotttnesss
elif attribute == 'ArtistID'.lower():
csvRowString += "\"" + song.artistID + "\""
elif attribute == 'ArtistLatitude'.lower():
latitude = song.artistLatitude
if latitude == 'nan':
latitude = ''
csvRowString += latitude
elif attribute == 'ArtistLocation'.lower():
location = song.artistLocation
location = location.replace(',', '')
csvRowString += "\"" + location + "\""
elif attribute == 'ArtistLongitude'.lower():
longitude = song.artistLongitude
if longitude == 'nan':
longitude = ''
csvRowString += longitude
elif attribute == 'ArtistName'.lower():
csvRowString += "\"" + song.artistName + "\""
elif attribute == 'BarsConfidence'.lower():
csvRowString += song.barsConfidence
elif attribute == 'BarsStart'.lower():
csvRowString += song.barsStart
elif attribute == 'BeatsConfidence'.lower():
csvRowString += song.beatsConfidence
elif attribute == 'BeatsStart'.lower():
csvRowString += song.beatsStart
elif attribute == 'Danceability'.lower():
csvRowString += song.danceability
elif attribute == 'Duration'.lower():
csvRowString += song.duration
elif attribute == 'EndOfFadeIn'.lower():
csvRowString += song.endOfFadeIn
elif attribute == 'Energy'.lower():
csvRowString += song.energy
elif attribute == 'KeySignature'.lower():
csvRowString += song.keySignature
elif attribute == 'KeySignatureConfidence'.lower():
# print "key sig conf: " + song.timeSignatureConfidence
csvRowString += song.keySignatureConfidence
elif attribute == 'Loudness'.lower():
csvRowString += song.loudness
elif attribute == 'Mode'.lower():
csvRowString += song.mode
elif attribute == 'ModeConfidence'.lower():
csvRowString += song.modeConfidence
elif attribute == 'SectionsConfidence'.lower():
csvRowString += song.sectionsConfidence
elif attribute == 'SectionsStart'.lower():
csvRowString += song.sectionsStart
elif attribute == 'SegmentsConfidence'.lower():
csvRowString += song.segmentsConfidence
elif attribute == 'SegmentsLoudnessMax'.lower():
csvRowString += song.segmentsLoudnessMax
elif attribute == 'SegmentsLoudnessMaxTime'.lower():
csvRowString += song.segmentsLoudnessMaxTime
elif attribute == 'SegmentsLoudnessMaxStart'.lower():
csvRowString += song.segmentsLoudnessMaxStart
elif attribute == 'SegmentsPitches'.lower():
csvRowString += song.segmentsPitches
elif attribute == 'SegmentsStart'.lower():
csvRowString += song.segmentsStart
elif attribute == 'SegmentsTimbre'.lower():
csvRowString += song.segmentsTimbre
elif attribute == 'SongHotttnesss'.lower():
csvRowString += song.hotttnesss
elif attribute == 'SongID'.lower():
csvRowString += "\"" + song.id + "\""
elif attribute == 'StartOfFadeOut'.lower():
csvRowString += song.startOfFadeOut
elif attribute == 'TatumsConfidence'.lower():
csvRowString += song.tatumsConfidence
elif attribute == 'TatumsStart'.lower():
csvRowString += song.tatumsStart
elif attribute == 'Tempo'.lower():
# print "Tempo: " + song.tempo
csvRowString += song.tempo
elif attribute == 'TimeSignature'.lower():
csvRowString += song.timeSignature
elif attribute == 'TimeSignatureConfidence'.lower():
# print "time sig conf: " + song.timeSignatureConfidence
csvRowString += song.timeSignatureConfidence
elif attribute == 'Title'.lower():
csvRowString += "\"" + song.title + "\""
elif attribute == 'Year'.lower():
csvRowString += song.year
else:
csvRowString += "Erm. This didn't work. Error. :( :(\n"
csvRowString += ","
#Remove the final comma from each row in the csv
lastIndex = len(csvRowString)
csvRowString = csvRowString[0:lastIndex - 1]
csvRowString += "\n"
outputFile1.write(csvRowString)
csvRowString = ""
songH5File.close()
outputFile1.close()
def _get_one_pitches(f):
with hdf5_getters.open_h5_file_read(f) as h5:
pitches = hdf5_getters.get_segments_pitches(h5)
return pitches
target_genres = ['house','techno','drum and bass','drum n bass','drum\'n\'bass',\
'drumnbass','drum \'n\' bass','jungle','breakbeat','trance','dubstep','trap','downtempo','disco',\
'industrial','synthpop','idm','idm - intelligent dance music','8-bit','hip-hop','hip hop','ambient','pop']
json_flat_file_metadata_name = '/Users/matthewsilver/Documents/MATLAB/msd_song_metadata.txt'
json_flat_file_name = '/Users/matthewsilver/Documents/MATLAB/lastfm_subset_flatfile.txt'
json_contents = (open(json_flat_file_name,'r')).read()
start_time = time.time()
song_count = 0
song_dict = dict()
basedir = '/Users/matthewsilver/Documents/MATLAB/MillionSongSubset/data/'
for root, dirs, files in os.walk(basedir):
for f in files:
if f[-3:] == '.h5':
h5 = hdf5_getters.open_h5_file_read(root + '/' + f)
track_id = hdf5_getters.get_track_id(h5)
track_id_name = '\"track_id\": \"{0}\"'.format(track_id)
if track_id in json_contents:
song_count += 1
print 'song {0}: {1}'.format(str(song_count),f)
# stupidly hacky way to get a json object from a regex search
# but I haven't figured out how to do it quicker
json_object = re.search('{\"%s\".*?}' % track_id,json_contents).group(0)
json_object = json.loads(json_object)
best_tag = ''
for t in [tag[0].lower() for tag in json_object[track_id]]:
if t in target_genres:
best_tag = t
file_counter = 0
for root, dirs, files in os.walk(base_dir):
fs = glob.glob(os.path.join(root, '*' + ext))
for f in fs:
counter += 1
if (counter % 1000) == 0:
print "Progress: {0}".format(counter)
if (counter % 10000) == 0:
outputFile1.close()
file_counter += 1
outputFile1 = open(
os.path.join(output_dir,
base_file_name).format(file_counter), 'w')
writeheader(outputFile1, csvHeaderString)
songH5File = hdf5_getters.open_h5_file_read(f)
song = Song(str(hdf5_getters.get_song_id(songH5File)))
testDanceability = hdf5_getters.get_danceability(songH5File)
# print type(testDanceability)
# print ("Here is the danceability: ") + str(testDanceability)
song.artistName = remove_trap_characters(
str(hdf5_getters.get_artist_name(songH5File)))
song.artistID = remove_trap_characters(
str(hdf5_getters.get_artist_id(songH5File)))
song.albumID = remove_trap_characters(
str(hdf5_getters.get_release_7digitalid(songH5File)))
song.artistLatitude = remove_trap_characters(
str(hdf5_getters.get_artist_latitude(songH5File)))
# Replace the comma in the location (if there is one), since this will displace the entire row