def extractEchoFeatures(directory, path, meta):
echo = nest.initialize()
# Retrieves Echo Nest summary
summary = nest.retrieveSummary(echo, path)
# Retrieves Echo Nest analysis
URL = summary[u"analysis_url"]
analysis = nest.retrieveAnalysis(URL)
# Writes summary data
meta = getSummaryMetadata(meta)
summary = nest.getCleanSummary(summary, analysis, meta, path)
filename = u"{}/{}".format(directory, u"info.json")
utils.writeJSON(summary, filename)
# Writes analysis data
analysis = nest.getCleanAnalysis(analysis)
for key, value in analysis.items():
filename = u"{}/{}".format(directory, key)
utils.writeData(value, u"{}.tsv".format(filename))
def extractEchoFeatures(directory, path, meta):
echo = nest.initialize()
# Retrieves Echo Nest summary
summary = nest.retrieveSummary(echo, path)
# Retrieves Echo Nest analysis
URL = summary[u"analysis_url"]
analysis = nest.retrieveAnalysis(URL)
# Writes summary data
meta = getSummaryMetadata(meta)
summary = nest.getCleanSummary(summary, analysis, meta, path)
filename = u"{}/{}".format(directory, u"info.json")
utils.writeJSON(summary, filename)
# Writes analysis data
analysis = nest.getCleanAnalysis(analysis)
for key, value in analysis.items():
filename = u"{}/{}".format(directory, key)
utils.writeData(value, u"{}.tsv".format(filename))
def extractLocalFeatures(directory, path):
input = []
# Preparation
audio = mir.load(path)
onsets, frames, times, spectra = prepareInputs(audio)
# Timbre
input.append(mir.getCleanMFCC(spectra))
input.append(mir.getCleanZC(frames))
# Rhythm
input.append(mir.getCleanTimes(times))
# Harmony
input.append(mir.getCleanNotes(spectra))
input.append(mir.getCleanChords(spectra))
# Loudness
input.append(mir.getCleanRMS(frames))
features = utils.combineLists(input)
filename = u"{}/{}".format(directory, u"features.tsv")
utils.writeData(features, filename)
def extractLocalFeatures(directory, path):
input = []
# Preparation
audio = mir.load(path)
onsets, frames, times, spectra = prepareInputs(audio)
# Timbre
input.append(mir.getCleanMFCC(spectra))
input.append(mir.getCleanZC(frames))
# Rhythm
input.append(mir.getCleanTimes(times))
# Harmony
input.append(mir.getCleanNotes(spectra))
input.append(mir.getCleanChords(spectra))
# Loudness
input.append(mir.getCleanRMS(frames))
features = utils.combineLists(input)
filename = u"{}/{}".format(directory, u"features.tsv")
utils.writeData(features, filename)
P_pred, rho_pred, u_pred, v_pred, Et_pred = model.predict(
P_back_test, x_test, y_test)
# #Error
error_P = np.linalg.norm(P_test - P_pred, 2) / np.linalg.norm(P_test, 2)
print("Test Error in P: " + str(error_P))
error_rho = np.linalg.norm(rho_test - rho_pred, 2) / np.linalg.norm(
rho_test, 2)
print("Test Error in rho: " + str(error_rho))
error_u = np.linalg.norm(u_test - u_pred, 2) / np.linalg.norm(u_test, 2)
print("Test Error in u: " + str(error_u))
error_v = np.linalg.norm(v_test - v_pred, 2) / np.linalg.norm(v_test, 2)
print("Test Error in v: " + str(error_v))
error_Et = np.linalg.norm(Et_test - Et_pred, 2) / np.linalg.norm(Et_test, 2)
print("Test Error in E: " + str(error_Et))
P_pred *= P_norm
rho_pred *= rho_norm
u_pred *= u_norm
v_pred *= v_norm
Et_pred *= Et_norm
path = os.getcwd() + '/predict/%s_bp=%s.csv' % (
setting_name, str(int(P_back_test[0] * P_norm)))
utils.writeData(path, x_test, y_test, P_pred, rho_pred, u_pred, v_pred,
Et_pred)
path2 = os.getcwd() + '/predict/%s_bp=%s_loss.csv' % (
setting_name, str(int(P_back_test[0] * P_norm)))
utils.writeLoss(path2, model.loss_vector, model.step_vector)
theirAlgosPredictions = []
for row in entireDataset:
tweet = row[5]
# set default sentiment score to 2 (neutral)
sentimentScore = 2
# try to find a positive emoticon
for emoticon in emoticons['4']:
if emoticon in tweet:
sentimentScore = 4
# try to find a negative emoticon
for emoticon in emoticons['0']:
if emoticon in tweet:
# if we have already found a positive emoticon for this message, call it neutral
if sentimentScore == 4:
sentimentScore = 2
else:
sentimentScore = 0
# add their algorithm's prediction as the first item in each row
row.insert(0, sentimentScore)
headerRow = ['Their algorithm\'s predictions','Manually Scored Sentiment','ID','Date','Search term used to gather tweet','User','Text']
entireDataset.insert(0, headerRow)
utils.writeData(entireDataset, 'testdata.with.their.algos.predictions.csv')
store = [data, 1]
elif btw_hold(data, store[0]):
store = ave_func(store, [data, 1])
else:
if store[1] <= winsize:
win.append(store)
else:
if len(win) > 0:
for w in win:
store[1] += w[1]
win = []
fs.append(map(int, store))
store = [data, 1]
if store is not None:
fs.append(map(int, store))
results = []
index = 0
for f in fs:
if len(results) == 0 or f[0] != results[-1][1]:
results.append((indexes[index], f[0]))
index += f[1]
return results
if __name__ == "__main__":
if len(sys.argv) != 5:
utils.printUsage(("datafile", "threshold", "winsize", "outputfile"))
results = smooth(utils.readData(sys.argv[1], int), float(sys.argv[2]),
int(sys.argv[3]))
utils.writeData(sys.argv[4], results, '%d\t%d\n')
ntcNoEmotFeatures, ntcNoEmotSentiment = nltkTwitterNoEmoticonsCorpus.getFeatures(
numWordsToUse)
ntcNoEmotTestTweets = nltkTwitterNoEmoticonsCorpus.formatTestData(testTweets)
ntcNoEmotEnsembleTweets = nltkTwitterNoEmoticonsCorpus.formatTestData(
ensembleTweets)
ntcNoEmotPredictions, ntcNoEmotEnsemblePredictions = trainClassifiers.trainClassifier(
ntcNoEmotFeatures, ntcNoEmotSentiment, ntcNoEmotTestTweets,
ntcNoEmotEnsembleTweets, ensembleSentiment)
# aggregate all our predictions together into a single matrix that we will train our ensemble classifier on
ensembledPredictions = utils.aggregatePredictions(
stsEnsemblePredictions, movieReviewEnsemblePredictions,
atcEnsemblePredictions, ntcEnsemblePredictions,
ntcNoEmotEnsemblePredictions, 'ensembleData.all.predictions.csv')
# do the same for our test data
testPredictions = utils.aggregatePredictions(stsPredictions,
movieReviewPredictions,
atcPredictions, ntcPredictions,
ntcNoEmotPredictions,
'testdata.all.predictions.csv')
# train the ensemble classifier on our ensembleData
# this will return a matrix with all the stage 1 classifiers' predictions on the test data, as well as the final predictions the ensemble algorithm produces
finalPredictions = trainClassifiers.trainEnsembleClassifier(
ensembledPredictions, ensembleSentiment, testPredictions)
utils.writeData(finalPredictions, 'testdata.entire.ensembled.predictions.csv')
for row in entireDataset:
tweet = row[5]
# set default sentiment score to 2 (neutral)
sentimentScore = 2
# try to find a positive emoticon
for emoticon in emoticons['4']:
if emoticon in tweet:
sentimentScore = 4
# try to find a negative emoticon
for emoticon in emoticons['0']:
if emoticon in tweet:
# if we have already found a positive emoticon for this message, call it neutral
if sentimentScore == 4:
sentimentScore = 2
else:
sentimentScore = 0
# add their algorithm's prediction as the first item in each row
row.insert(0, sentimentScore)
headerRow = [
'Their algorithm\'s predictions', 'Manually Scored Sentiment', 'ID',
'Date', 'Search term used to gather tweet', 'User', 'Text'
]
entireDataset.insert(0, headerRow)
utils.writeData(entireDataset, 'testdata.with.their.algos.predictions.csv')
ntcPredictions, ntcEnsemblePredictions = trainClassifiers.trainClassifier(ntcFeatures, ntcSentiment, ntcTestTweets, ntcEnsembleTweets, ensembleSentiment) ############################################################## # NLTK Twitter Corpus without emoticons (ntcNoEmot) ############################################################## ntcNoEmotFeatures, ntcNoEmotSentiment = nltkTwitterNoEmoticonsCorpus.getFeatures(numWordsToUse) ntcNoEmotTestTweets = nltkTwitterNoEmoticonsCorpus.formatTestData(testTweets) ntcNoEmotEnsembleTweets = nltkTwitterNoEmoticonsCorpus.formatTestData(ensembleTweets) ntcNoEmotPredictions, ntcNoEmotEnsemblePredictions = trainClassifiers.trainClassifier(ntcNoEmotFeatures, ntcNoEmotSentiment, ntcNoEmotTestTweets, ntcNoEmotEnsembleTweets, ensembleSentiment) # aggregate all our predictions together into a single matrix that we will train our ensemble classifier on ensembledPredictions = utils.aggregatePredictions(stsEnsemblePredictions, movieReviewEnsemblePredictions, atcEnsemblePredictions, ntcEnsemblePredictions, ntcNoEmotEnsemblePredictions, 'ensembleData.all.predictions.csv') # do the same for our test data testPredictions = utils.aggregatePredictions(stsPredictions, movieReviewPredictions, atcPredictions, ntcPredictions, ntcNoEmotPredictions, 'testdata.all.predictions.csv') # train the ensemble classifier on our ensembleData # this will return a matrix with all the stage 1 classifiers' predictions on the test data, as well as the final predictions the ensemble algorithm produces finalPredictions = trainClassifiers.trainEnsembleClassifier(ensembledPredictions, ensembleSentiment, testPredictions) utils.writeData(finalPredictions, 'testdata.entire.ensembled.predictions.csv')