def classifyFolderWrapper(inputFolder, modelType, modelName, outputMode=False):
if not os.path.isfile(modelName):
raise Exception("Input modelName not found!")
if modelType=='svm':
[Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model(modelName)
elif modelType=='knn':
[Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model_knn(modelName)
PsAll = numpy.zeros((len(classNames), ))
files = "*.wav"
if os.path.isdir(inputFolder):
strFilePattern = os.path.join(inputFolder, files)
else:
strFilePattern = inputFolder + files
wavFilesList = []
wavFilesList.extend(glob.glob(strFilePattern))
wavFilesList = sorted(wavFilesList)
if len(wavFilesList)==0:
print "No WAV files found!"
return
Results = []
for wavFile in wavFilesList:
[Fs, x] = audioBasicIO.readAudioFile(wavFile)
signalLength = x.shape[0] / float(Fs)
[Result, P, classNames] = aT.fileClassification(wavFile, modelName, modelType)
PsAll += (numpy.array(P) * signalLength)
Result = int(Result)
Results.append(Result)
if outputMode:
print "{0:s}\t{1:s}".format(wavFile,classNames[Result])
Results = numpy.array(Results)
# print distribution of classes:
[Histogram, _] = numpy.histogram(Results, bins=numpy.arange(len(classNames)+1))
if outputMode:
for i,h in enumerate(Histogram):
print "{0:20s}\t\t{1:d}".format(classNames[i], h)
PsAll = PsAll / numpy.sum(PsAll)
if outputMode:
fig = plt.figure()
ax = fig.add_subplot(111)
plt.title("Classes percentage " + inputFolder.replace('Segments',''))
ax.axis((0, len(classNames)+1, 0, 1))
ax.set_xticks(numpy.array(range(len(classNames)+1)))
ax.set_xticklabels([" "] + classNames)
ax.bar(numpy.array(range(len(classNames)))+0.5, PsAll)
plt.show()
return classNames, PsAll
def getMusicSegmentsFromFile(inputFile):
modelType = "svm"
modelName = "data/svmMovies8classes"
dirOutput = inputFile[0:-4] + "_musicSegments"
if os.path.exists(dirOutput) and dirOutput != ".":
shutil.rmtree(dirOutput)
os.makedirs(dirOutput)
[Fs, x] = audioBasicIO.readAudioFile(inputFile)
if modelType == 'svm':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model(modelName)
elif modelType == 'knn':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model_knn(modelName)
flagsInd, classNames, acc, CM = aS.mtFileClassification(inputFile,
modelName,
modelType,
plotResults=False,
gtFile="")
segs, classes = aS.flags2segs(flagsInd, mtStep)
for i, s in enumerate(segs):
if (classNames[int(classes[i])]
== "Music") and (s[1] - s[0] >= minDuration):
strOut = "{0:s}{1:.3f}-{2:.3f}.wav".format(dirOutput + os.sep,
s[0], s[1])
wavfile.write(strOut, Fs, x[int(Fs * s[0]):int(Fs * s[1])])
def getMusicSegmentsFromFile(inputFile):
modelType = "svm"
modelName = "data/svmMovies8classes"
dirOutput = inputFile[0:-4] + "_musicSegments"
if os.path.exists(dirOutput) and dirOutput!=".":
shutil.rmtree(dirOutput)
os.makedirs(dirOutput)
[Fs, x] = audioBasicIO.readAudioFile(inputFile)
if modelType=='svm':
[Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model(modelName)
elif modelType=='knn':
[Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model_knn(modelName)
flagsInd, classNames, acc, CM = aS.mtFileClassification(inputFile, modelName, modelType, plotResults = False, gtFile = "")
segs, classes = aS.flags2segs(flagsInd, mtStep)
for i, s in enumerate(segs):
if (classNames[int(classes[i])] == "Music") and (s[1] - s[0] >= minDuration):
strOut = "{0:s}{1:.3f}-{2:.3f}.wav".format(dirOutput+os.sep, s[0], s[1])
wavfile.write( strOut, Fs, x[int(Fs*s[0]):int(Fs*s[1])])
def classifyFolderWrapper(inputFolder, modelType, modelName, outputMode=False):
if not os.path.isfile(modelName):
raise Exception("Input modelName not found!")
if modelType == 'svm':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model(modelName)
elif modelType == 'knn':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model_knn(modelName)
PsAll = numpy.zeros((len(classNames), ))
files = "*.wav"
if os.path.isdir(inputFolder):
strFilePattern = os.path.join(inputFolder, files)
else:
strFilePattern = inputFolder + files
wavFilesList = []
wavFilesList.extend(glob.glob(strFilePattern))
wavFilesList = sorted(wavFilesList)
if len(wavFilesList) == 0:
print "No WAV files found!"
return
Results = []
for wavFile in wavFilesList:
[Fs, x] = audioBasicIO.readAudioFile(wavFile)
signalLength = x.shape[0] / float(Fs)
[Result, P,
classNames] = aT.file_classification(wavFile, modelName, modelType)
PsAll += (numpy.array(P) * signalLength)
Result = int(Result)
Results.append(Result)
if outputMode:
print "{0:s}\t{1:s}".format(wavFile, classNames[Result])
Results = numpy.array(Results)
# print distribution of classes:
[Histogram, _] = numpy.histogram(Results,
bins=numpy.arange(len(classNames) + 1))
if outputMode:
for i, h in enumerate(Histogram):
print "{0:20s}\t\t{1:d}".format(classNames[i], h)
PsAll = PsAll / numpy.sum(PsAll)
if outputMode:
fig = plt.figure()
ax = fig.add_subplot(111)
plt.title("Classes percentage " + inputFolder.replace('Segments', ''))
ax.axis((0, len(classNames) + 1, 0, 1))
ax.set_xticks(numpy.array(range(len(classNames) + 1)))
ax.set_xticklabels([" "] + classNames)
ax.bar(numpy.array(range(len(classNames))) + 0.5, PsAll)
plt.show()
return classNames, PsAll
def speakerDiarization(filename,
n_speakers,
mt_size=2.0,
mt_step=0.2,
st_win=0.05,
lda_dim=35,
plot_res=False):
'''
ARGUMENTS:
- filename: the name of the WAV file to be analyzed
- n_speakers the number of speakers (clusters) in the recording (<=0 for unknown)
- mt_size (opt) mid-term window size
- mt_step (opt) mid-term window step
- st_win (opt) short-term window size
- lda_dim (opt) LDA dimension (0 for no LDA)
- plot_res (opt) 0 for not plotting the results 1 for plottingy
'''
[fs, x] = audioBasicIO.readAudioFile(filename)
x = audioBasicIO.stereo2mono(x)
duration = len(x) / fs
# [classifier_1, MEAN1, STD1, classNames1, mtWin1, mtStep1, stWin1, stStep1, computeBEAT1] = aT.load_model_knn(os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "knnSpeakerAll"))
# [classifier_2, MEAN2, STD2, classNames2, mtWin2, mtStep2, stWin2, stStep2, computeBEAT2] = aT.load_model_knn(os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", "knnSpeakerFemaleMale"))
[
classifier_1, MEAN1, STD1, classNames1, mtWin1, mtStep1, stWin1,
stStep1, computeBEAT1
] = aT.load_model_knn("data/knnSpeakerAll")
[
classifier_2, MEAN2, STD2, classNames2, mtWin2, mtStep2, stWin2,
stStep2, computeBEAT2
] = aT.load_model_knn("data/knnSpeakerFemaleMale")
[mt_feats, st_feats, _] = aF.mtFeatureExtraction(x, fs, mt_size * fs,
mt_step * fs,
round(fs * st_win),
round(fs * st_win * 0.5))
MidTermFeatures2 = numpy.zeros(
(mt_feats.shape[0] + len(classNames1) + len(classNames2),
mt_feats.shape[1]))
for i in range(mt_feats.shape[1]):
cur_f1 = (mt_feats[:, i] - MEAN1) / STD1
cur_f2 = (mt_feats[:, i] - MEAN2) / STD2
[res, P1] = aT.classifierWrapper(classifier_1, "knn", cur_f1)
[res, P2] = aT.classifierWrapper(classifier_2, "knn", cur_f2)
MidTermFeatures2[0:mt_feats.shape[0], i] = mt_feats[:, i]
MidTermFeatures2[mt_feats.shape[0]:mt_feats.shape[0] +
len(classNames1), i] = P1 + 0.0001
MidTermFeatures2[mt_feats.shape[0] + len(classNames1)::,
i] = P2 + 0.0001
mt_feats = MidTermFeatures2 # TODO
iFeaturesSelect = [
8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 41, 42, 43, 44, 45,
46, 47, 48, 49, 50, 51, 52, 53
]
mt_feats = mt_feats[iFeaturesSelect, :]
(mt_feats_norm, MEAN, STD) = aT.normalizeFeatures([mt_feats.T])
mt_feats_norm = mt_feats_norm[0].T
n_wins = mt_feats.shape[1]
# remove outliers:
dist_all = numpy.sum(distance.squareform(distance.pdist(mt_feats_norm.T)),
axis=0)
m_dist_all = numpy.mean(dist_all)
i_non_outliers = numpy.nonzero(dist_all < 1.2 * m_dist_all)[0]
# TODO: Combine energy threshold for outlier removal:
#EnergyMin = numpy.min(mt_feats[1,:])
#EnergyMean = numpy.mean(mt_feats[1,:])
#Thres = (1.5*EnergyMin + 0.5*EnergyMean) / 2.0
#i_non_outliers = numpy.nonzero(mt_feats[1,:] > Thres)[0]
#print i_non_outliers
perOutLier = (100.0 * (n_wins - i_non_outliers.shape[0])) / n_wins
mt_feats_norm_or = mt_feats_norm
mt_feats_norm = mt_feats_norm[:, i_non_outliers]
# LDA dimensionality reduction:
if lda_dim > 0:
#[mt_feats_to_red, _, _] = aF.mtFeatureExtraction(x, fs, mt_size * fs, st_win * fs, round(fs*st_win), round(fs*st_win));
# extract mid-term features with minimum step:
mt_win_ratio = int(round(mt_size / st_win))
mt_step_ratio = int(round(st_win / st_win))
mt_feats_to_red = []
num_of_features = len(st_feats)
num_of_stats = 2
#for i in range(num_of_stats * num_of_features + 1):
for i in range(num_of_stats * num_of_features):
mt_feats_to_red.append([])
for i in range(
num_of_features): # for each of the short-term features:
curPos = 0
N = len(st_feats[i])
while (curPos < N):
N1 = curPos
N2 = curPos + mt_win_ratio
if N2 > N:
N2 = N
curStFeatures = st_feats[i][N1:N2]
mt_feats_to_red[i].append(numpy.mean(curStFeatures))
mt_feats_to_red[i + num_of_features].append(
numpy.std(curStFeatures))
curPos += mt_step_ratio
mt_feats_to_red = numpy.array(mt_feats_to_red)
mt_feats_to_red_2 = numpy.zeros(
(mt_feats_to_red.shape[0] + len(classNames1) + len(classNames2),
mt_feats_to_red.shape[1]))
for i in range(mt_feats_to_red.shape[1]):
cur_f1 = (mt_feats_to_red[:, i] - MEAN1) / STD1
cur_f2 = (mt_feats_to_red[:, i] - MEAN2) / STD2
[res, P1] = aT.classifierWrapper(classifier_1, "knn", cur_f1)
[res, P2] = aT.classifierWrapper(classifier_2, "knn", cur_f2)
mt_feats_to_red_2[0:mt_feats_to_red.shape[0],
i] = mt_feats_to_red[:, i]
mt_feats_to_red_2[
mt_feats_to_red.shape[0]:mt_feats_to_red.shape[0] +
len(classNames1), i] = P1 + 0.0001
mt_feats_to_red_2[mt_feats_to_red.shape[0] + len(classNames1)::,
i] = P2 + 0.0001
mt_feats_to_red = mt_feats_to_red_2
mt_feats_to_red = mt_feats_to_red[iFeaturesSelect, :]
#mt_feats_to_red += numpy.random.rand(mt_feats_to_red.shape[0], mt_feats_to_red.shape[1]) * 0.0000010
(mt_feats_to_red, MEAN,
STD) = aT.normalizeFeatures([mt_feats_to_red.T])
mt_feats_to_red = mt_feats_to_red[0].T
#dist_all = numpy.sum(distance.squareform(distance.pdist(mt_feats_to_red.T)), axis=0)
#m_dist_all = numpy.mean(dist_all)
#iNonOutLiers2 = numpy.nonzero(dist_all < 3.0*m_dist_all)[0]
#mt_feats_to_red = mt_feats_to_red[:, iNonOutLiers2]
Labels = numpy.zeros((mt_feats_to_red.shape[1], ))
LDAstep = 1.0
LDAstepRatio = LDAstep / st_win
#print LDAstep, LDAstepRatio
for i in range(Labels.shape[0]):
Labels[i] = int(i * st_win / LDAstepRatio)
clf = sklearn.discriminant_analysis.LinearDiscriminantAnalysis(
n_components=lda_dim)
clf.fit(mt_feats_to_red.T, Labels)
mt_feats_norm = (clf.transform(mt_feats_norm.T)).T
if n_speakers <= 0:
s_range = range(2, 10)
else:
s_range = [n_speakers]
clsAll = []
sil_all = []
centersAll = []
for iSpeakers in s_range:
k_means = sklearn.cluster.KMeans(n_clusters=iSpeakers)
k_means.fit(mt_feats_norm.T)
cls = k_means.labels_
means = k_means.cluster_centers_
# Y = distance.squareform(distance.pdist(mt_feats_norm.T))
clsAll.append(cls)
centersAll.append(means)
sil_1 = []
sil_2 = []
for c in range(iSpeakers):
# for each speaker (i.e. for each extracted cluster)
clust_per_cent = numpy.nonzero(cls == c)[0].shape[0] / \
float(len(cls))
if clust_per_cent < 0.020:
sil_1.append(0.0)
sil_2.append(0.0)
else:
# get subset of feature vectors
mt_feats_norm_temp = mt_feats_norm[:, cls == c]
# compute average distance between samples
# that belong to the cluster (a values)
Yt = distance.pdist(mt_feats_norm_temp.T)
sil_1.append(numpy.mean(Yt) * clust_per_cent)
silBs = []
for c2 in range(iSpeakers):
# compute distances from samples of other clusters
if c2 != c:
clust_per_cent_2 = numpy.nonzero(cls == c2)[0].shape[0] /\
float(len(cls))
MidTermFeaturesNormTemp2 = mt_feats_norm[:, cls == c2]
Yt = distance.cdist(mt_feats_norm_temp.T,
MidTermFeaturesNormTemp2.T)
silBs.append(
numpy.mean(Yt) *
(clust_per_cent + clust_per_cent_2) / 2.0)
silBs = numpy.array(silBs)
# ... and keep the minimum value (i.e.
# the distance from the "nearest" cluster)
sil_2.append(min(silBs))
sil_1 = numpy.array(sil_1)
sil_2 = numpy.array(sil_2)
sil = []
for c in range(iSpeakers):
# for each cluster (speaker) compute silhouette
sil.append(
(sil_2[c] - sil_1[c]) / (max(sil_2[c], sil_1[c]) + 0.00001))
# keep the AVERAGE SILLOUETTE
sil_all.append(numpy.mean(sil))
imax = numpy.argmax(sil_all)
# optimal number of clusters
nSpeakersFinal = s_range[imax]
# generate the final set of cluster labels
# (important: need to retrieve the outlier windows:
# this is achieved by giving them the value of their
# nearest non-outlier window)
cls = numpy.zeros((n_wins, ))
for i in range(n_wins):
j = numpy.argmin(numpy.abs(i - i_non_outliers))
cls[i] = clsAll[imax][j]
# Post-process method 1: hmm smoothing
for i in range(1):
# hmm training
start_prob, transmat, means, cov = \
trainHMM_computeStatistics(mt_feats_norm_or, cls)
hmm = hmmlearn.hmm.GaussianHMM(start_prob.shape[0], "diag")
hmm.startprob_ = start_prob
hmm.transmat_ = transmat
hmm.means_ = means
hmm.covars_ = cov
cls = hmm.predict(mt_feats_norm_or.T)
# Post-process method 2: median filtering:
cls = scipy.signal.medfilt(cls, 13)
cls = scipy.signal.medfilt(cls, 11)
sil = sil_all[imax]
class_names = ["speaker{0:d}".format(c) for c in range(nSpeakersFinal)]
# load ground-truth if available
gt_file = filename.replace('.wav', '.segments')
# if groundturh exists
if os.path.isfile(gt_file):
[seg_start, seg_end, seg_labs] = readSegmentGT(gt_file)
flags_gt, class_names_gt = segs2flags(seg_start, seg_end, seg_labs,
mt_step)
if plot_res:
fig = plt.figure()
if n_speakers > 0:
ax1 = fig.add_subplot(111)
else:
ax1 = fig.add_subplot(211)
ax1.set_yticks(numpy.array(range(len(class_names))))
ax1.axis((0, duration, -1, len(class_names)))
ax1.set_yticklabels(class_names)
ax1.plot(numpy.array(range(len(cls))) * mt_step + mt_step / 2.0, cls)
if os.path.isfile(gt_file):
if plot_res:
ax1.plot(
numpy.array(range(len(flags_gt))) * mt_step + mt_step / 2.0,
flags_gt, 'r')
purity_cluster_m, purity_speaker_m = \
evaluateSpeakerDiarization(cls, flags_gt)
print("{0:.1f}\t{1:.1f}".format(100 * purity_cluster_m,
100 * purity_speaker_m))
if plot_res:
plt.title("Cluster purity: {0:.1f}% - "
"Speaker purity: {1:.1f}%".format(
100 * purity_cluster_m, 100 * purity_speaker_m))
if plot_res:
plt.xlabel("time (seconds)")
#print s_range, sil_all
if n_speakers <= 0:
plt.subplot(212)
plt.plot(s_range, sil_all)
plt.xlabel("number of clusters")
plt.ylabel("average clustering's sillouette")
#plt.show()
plt.savefig('output/outImg.jpg')
return cls
def mtFileClassification(input_file,
model_name,
model_type,
plot_results=False,
gt_file=""):
'''
This function performs mid-term classification of an audio stream.
Towards this end, supervised knowledge is used, i.e. a pre-trained classifier.
ARGUMENTS:
- input_file: path of the input WAV file
- model_name: name of the classification model
- model_type: svm or knn depending on the classifier type
- plot_results: True if results are to be plotted using
matplotlib along with a set of statistics
RETURNS:
- segs: a sequence of segment's endpoints: segs[i] is the
endpoint of the i-th segment (in seconds)
- classes: a sequence of class flags: class[i] is the
class ID of the i-th segment
'''
if not os.path.isfile(model_name):
print("mtFileClassificationError: input model_type not found!")
return (-1, -1, -1, -1)
# Load classifier:
if model_type == "knn":
[classifier, MEAN, STD, class_names, mt_win, mt_step, st_win, st_step, compute_beat] = \
aT.load_model_knn(model_name)
else:
[
classifier, MEAN, STD, class_names, mt_win, mt_step, st_win,
st_step, compute_beat
] = aT.load_model(model_name)
if compute_beat:
print("Model " + model_name + " contains long-term music features "
"(beat etc) and cannot be used in "
"segmentation")
return (-1, -1, -1, -1)
[fs, x] = audioBasicIO.readAudioFile(input_file) # load input file
if fs == -1: # could not read file
return (-1, -1, -1, -1)
x = audioBasicIO.stereo2mono(x) # convert stereo (if) to mono
duration = len(x) / fs
# mid-term feature extraction:
[mt_feats, _, _] = aF.mtFeatureExtraction(x, fs, mt_win * fs, mt_step * fs,
round(fs * st_win),
round(fs * st_step))
flags = []
Ps = []
flags_ind = []
for i in range(
mt_feats.shape[1]
): # for each feature vector (i.e. for each fix-sized segment):
cur_fv = (mt_feats[:, i] -
MEAN) / STD # normalize current feature vector
[res, P] = aT.classifierWrapper(classifier, model_type,
cur_fv) # classify vector
flags_ind.append(res)
flags.append(class_names[int(res)]) # update class label matrix
Ps.append(numpy.max(P)) # update probability matrix
flags_ind = numpy.array(flags_ind)
# 1-window smoothing
for i in range(1, len(flags_ind) - 1):
if flags_ind[i - 1] == flags_ind[i + 1]:
flags_ind[i] = flags_ind[i + 1]
# convert fix-sized flags to segments and classes
(segs, classes) = flags2segs(flags, mt_step)
segs[-1] = len(x) / float(fs)
# Load grount-truth:
if os.path.isfile(gt_file):
[seg_start_gt, seg_end_gt, seg_l_gt] = readSegmentGT(gt_file)
flags_gt, class_names_gt = segs2flags(seg_start_gt, seg_end_gt,
seg_l_gt, mt_step)
flags_ind_gt = []
for j, fl in enumerate(flags_gt):
# "align" labels with GT
if class_names_gt[flags_gt[j]] in class_names:
flags_ind_gt.append(
class_names.index(class_names_gt[flags_gt[j]]))
else:
flags_ind_gt.append(-1)
flags_ind_gt = numpy.array(flags_ind_gt)
cm = numpy.zeros((len(class_names_gt), len(class_names_gt)))
for i in range(min(flags_ind.shape[0], flags_ind_gt.shape[0])):
cm[int(flags_ind_gt[i]), int(flags_ind[i])] += 1
else:
cm = []
flags_ind_gt = numpy.array([])
acc = plotSegmentationResults(flags_ind, flags_ind_gt, class_names,
mt_step, not plot_results)
if acc >= 0:
print("Overall Accuracy: {0:.3f}".format(acc))
return (flags_ind, class_names_gt, acc, cm)
else:
return (flags_ind, class_names, acc, cm)
def recordAnalyzeAudio(duration, outputWavFile, midTermBufferSizeSec, modelName, modelType): ''' recordAnalyzeAudio(duration, outputWavFile, midTermBufferSizeSec, modelName, modelType) This function is used to record and analyze audio segments, in a fix window basis. ARGUMENTS: - duration total recording duration - outputWavFile path of the output WAV file - midTermBufferSizeSec (fix)segment length in seconds - modelName classification model name - modelType classification model type ''' if modelType=='svm': [Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model(modelName) elif modelType=='knn': [Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep, compute_beat] = aT.load_model_knn(modelName) else: Classifier = None inp = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, alsaaudio.PCM_NONBLOCK) inp.setchannels(1) inp.setrate(Fs) inp.setformat(alsaaudio.PCM_FORMAT_S16_LE) inp.setperiodsize(512) midTermBufferSize = int(midTermBufferSizeSec * Fs) allData = [] midTermBuffer = [] curWindow = [] count = 0 while len(allData)<duration*Fs: # Read data from device l,data = inp.read() if l: for i in range(l): curWindow.append(audioop.getsample(data, 2, i)) if (len(curWindow)+len(midTermBuffer)>midTermBufferSize): samplesToCopyToMidBuffer = midTermBufferSize - len(midTermBuffer) else: samplesToCopyToMidBuffer = len(curWindow) midTermBuffer = midTermBuffer + curWindow[0:samplesToCopyToMidBuffer]; del(curWindow[0:samplesToCopyToMidBuffer]) if len(midTermBuffer) == midTermBufferSize: count += 1 if Classifier!=None: [mtFeatures, stFeatures, _] = aF.mtFeatureExtraction(midTermBuffer, Fs, 2.0*Fs, 2.0*Fs, 0.020*Fs, 0.020*Fs) curFV = (mtFeatures[:,0] - MEAN) / STD; [result, P] = aT.classifierWrapper(Classifier, modelType, curFV) print classNames[int(result)] allData = allData + midTermBuffer plt.clf() plt.plot(midTermBuffer) plt.show(block = False) plt.draw() midTermBuffer = [] allDataArray = numpy.int16(allData) wavfile.write(outputWavFile, Fs, allDataArray)
def recordAnalyzeAudio(duration, outputWavFile, midTermBufferSizeSec,
modelName, modelType):
'''
recordAnalyzeAudio(duration, outputWavFile, midTermBufferSizeSec, modelName, modelType)
This function is used to record and analyze audio segments, in a fix window basis.
ARGUMENTS:
- duration total recording duration
- outputWavFile path of the output WAV file
- midTermBufferSizeSec (fix)segment length in seconds
- modelName classification model name
- modelType classification model type
'''
if modelType == 'svm':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model(modelName)
elif modelType == 'knn':
[
Classifier, MEAN, STD, classNames, mtWin, mtStep, stWin, stStep,
compute_beat
] = aT.load_model_knn(modelName)
else:
Classifier = None
inp = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, alsaaudio.PCM_NONBLOCK)
inp.setchannels(1)
inp.setrate(Fs)
inp.setformat(alsaaudio.PCM_FORMAT_S16_LE)
inp.setperiodsize(512)
midTermBufferSize = int(midTermBufferSizeSec * Fs)
allData = []
midTermBuffer = []
curWindow = []
count = 0
while len(allData) < duration * Fs:
# Read data from device
l, data = inp.read()
if l:
for i in range(l):
curWindow.append(audioop.getsample(data, 2, i))
if (len(curWindow) + len(midTermBuffer) > midTermBufferSize):
samplesToCopyToMidBuffer = midTermBufferSize - len(
midTermBuffer)
else:
samplesToCopyToMidBuffer = len(curWindow)
midTermBuffer = midTermBuffer + curWindow[
0:samplesToCopyToMidBuffer]
del (curWindow[0:samplesToCopyToMidBuffer])
if len(midTermBuffer) == midTermBufferSize:
count += 1
if Classifier != None:
[mtFeatures, stFeatures,
_] = aF.mtFeatureExtraction(midTermBuffer, Fs, 2.0 * Fs,
2.0 * Fs, 0.020 * Fs, 0.020 * Fs)
curFV = (mtFeatures[:, 0] - MEAN) / STD
[result, P] = aT.classifier_wrapper(Classifier, modelType,
curFV)
print classNames[int(result)]
allData = allData + midTermBuffer
plt.clf()
plt.plot(midTermBuffer)
plt.show(block=False)
plt.draw()
midTermBuffer = []
allDataArray = numpy.int16(allData)
wavfile.write(outputWavFile, Fs, allDataArray)
def speaker_diarization(filename, n_speakers, mid_window=2.0, mid_step=0.2,
short_window=0.05, lda_dim=35, plot_res=False):
"""
ARGUMENTS:
- filename: the name of the WAV file to be analyzed
- n_speakers the number of speakers (clusters) in
the recording (<=0 for unknown)
- mid_window (opt) mid-term window size
- mid_step (opt) mid-term window step
- short_window (opt) short-term window size
- lda_dim (opt LDA dimension (0 for no LDA)
- plot_res (opt) 0 for not plotting the results 1 for plotting
"""
sampling_rate, signal = audioBasicIO.read_audio_file(filename)
signal = audioBasicIO.stereo_to_mono(signal)
duration = len(signal) / sampling_rate
base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"data/models")
classifier_all, mean_all, std_all, class_names_all, _, _, _, _, _ = \
at.load_model_knn(os.path.join(base_dir, "knn_speaker_10"))
classifier_fm, mean_fm, std_fm, class_names_fm, _, _, _, _, _ = \
at.load_model_knn(os.path.join(base_dir, "knn_speaker_male_female"))
mid_feats, st_feats, _ = \
mtf.mid_feature_extraction(signal, sampling_rate,
mid_window * sampling_rate,
mid_step * sampling_rate,
round(sampling_rate * short_window),
round(sampling_rate * short_window * 0.5))
mid_term_features = np.zeros((mid_feats.shape[0] + len(class_names_all) +
len(class_names_fm), mid_feats.shape[1]))
for index in range(mid_feats.shape[1]):
feature_norm_all = (mid_feats[:, index] - mean_all) / std_all
feature_norm_fm = (mid_feats[:, index] - mean_fm) / std_fm
_, p1 = at.classifier_wrapper(classifier_all, "knn", feature_norm_all)
_, p2 = at.classifier_wrapper(classifier_fm, "knn", feature_norm_fm)
start = mid_feats.shape[0]
end = mid_feats.shape[0] + len(class_names_all)
mid_term_features[0:mid_feats.shape[0], index] = mid_feats[:, index]
mid_term_features[start:end, index] = p1 + 1e-4
mid_term_features[end::, index] = p2 + 1e-4
mid_feats = mid_term_features # TODO
feature_selected = [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 41,
42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]
mid_feats = mid_feats[feature_selected, :]
mid_feats_norm, mean, std = at.normalize_features([mid_feats.T])
mid_feats_norm = mid_feats_norm[0].T
n_wins = mid_feats.shape[1]
dist_all = np.sum(distance.squareform(distance.pdist(mid_feats_norm.T)),
axis=0)
m_dist_all = np.mean(dist_all)
i_non_outliers = np.nonzero(dist_all < 1.2 * m_dist_all)[0]
mt_feats_norm_or = mid_feats_norm
mid_feats_norm = mid_feats_norm[:, i_non_outliers]
if n_speakers <= 0:
s_range = range(2, 10)
else:
s_range = [n_speakers]
cluster_labels = []
sil_all = []
cluster_centers = []
for speakers in s_range:
k_means = sklearn.cluster.KMeans(n_clusters=speakers)
k_means.fit(mid_feats_norm.T)
cls = k_means.labels_
means = k_means.cluster_centers_
cluster_labels.append(cls)
cluster_centers.append(means)
sil_1 = []; sil_2 = []
for c in range(speakers):
# for each speaker (i.e. for each extracted cluster)
clust_per_cent = np.nonzero(cls == c)[0].shape[0] / float(len(cls))
if clust_per_cent < 0.020:
sil_1.append(0.0)
sil_2.append(0.0)
else:
mt_feats_norm_temp = mid_feats_norm[:, cls == c]
dist = distance.pdist(mt_feats_norm_temp.T)
sil_1.append(np.mean(dist)*clust_per_cent)
sil_temp = []
for c2 in range(speakers):
if c2 != c:
clust_per_cent_2 = np.nonzero(cls == c2)[0].shape[0] /\
float(len(cls))
mid_features_temp = mid_feats_norm[:, cls == c2]
dist = distance.cdist(mt_feats_norm_temp.T,
mid_features_temp.T)
sil_temp.append(np.mean(dist)*(clust_per_cent
+ clust_per_cent_2)/2.0)
sil_temp = np.array(sil_temp)
sil_2.append(min(sil_temp))
sil_1 = np.array(sil_1)
sil_2 = np.array(sil_2)
sil = []
for c in range(speakers):
sil.append((sil_2[c] - sil_1[c]) / (max(sil_2[c], sil_1[c]) + 1e-5))
sil_all.append(np.mean(sil))
imax = int(np.argmax(sil_all))
num_speakers = s_range[imax]
cls = np.zeros((n_wins,))
for index in range(n_wins):
j = np.argmin(np.abs(index-i_non_outliers))
cls[index] = cluster_labels[imax][j]
for index in range(1):
start_prob, transmat, means, cov = \
train_hmm_compute_statistics(mt_feats_norm_or, cls)
hmm = hmmlearn.hmm.GaussianHMM(start_prob.shape[0], "diag")
hmm.startprob_ = start_prob
hmm.transmat_ = transmat
hmm.means_ = means; hmm.covars_ = cov
cls = hmm.predict(mt_feats_norm_or.T)
# Post-process method 2: median filtering:
cls = scipy.signal.medfilt(cls, 13)
cls = scipy.signal.medfilt(cls, 11)
class_names = ["speaker{0:d}".format(c) for c in range(num_speakers)]
if plot_res:
fig = plt.figure(figsize=(10, 4))
if n_speakers > 0:
ax1 = fig.add_subplot(111)
else:
ax1 = fig.add_subplot(211)
ax1.set_yticks(np.array(range(len(class_names))))
ax1.axis((0, duration, -1, len(class_names)))
ax1.set_yticklabels(class_names)
list_labels = np.array(range(len(cls))) * mid_step + mid_step
ax1.set_xticks(list_labels[::25])
ax1.plot(np.array(range(len(cls))) * mid_step + mid_step, cls)
if plot_res:
plt.xlabel("time (seconds)")
if n_speakers <= 0:
plt.subplot(212)
plt.plot(s_range, sil_all)
plt.xlabel("number of clusters")
plt.ylabel("average clustering's sillouette")
plt.savefig('foo.png')
return cls, sampling_rate, len(signal)
def speaker_diarization(filename, n_speakers, mid_window=2.0, mid_step=0.2,
short_window=0.05, lda_dim=35, plot_res=False):
"""
ARGUMENTS:
- filename: the name of the WAV file to be analyzed
- n_speakers the number of speakers (clusters) in
the recording (<=0 for unknown)
- mid_window (opt) mid-term window size
- mid_step (opt) mid-term window step
- short_window (opt) short-term window size
- lda_dim (opt LDA dimension (0 for no LDA)
- plot_res (opt) 0 for not plotting the results 1 for plotting
"""
sampling_rate, signal = audioBasicIO.read_audio_file(filename)
signal = audioBasicIO.stereo_to_mono(signal)
duration = len(signal) / sampling_rate
base_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"data/models")
classifier_all, mean_all, std_all, class_names_all, _, _, _, _, _ = \
at.load_model_knn(os.path.join(base_dir, "knn_speaker_10"))
classifier_fm, mean_fm, std_fm, class_names_fm, _, _, _, _, _ = \
at.load_model_knn(os.path.join(base_dir, "knn_speaker_male_female"))
mid_feats, st_feats, _ = \
mtf.mid_feature_extraction(signal, sampling_rate,
mid_window * sampling_rate,
mid_step * sampling_rate,
round(sampling_rate * short_window),
round(sampling_rate * short_window * 0.5))
mid_term_features = np.zeros((mid_feats.shape[0] + len(class_names_all) +
len(class_names_fm), mid_feats.shape[1]))
for index in range(mid_feats.shape[1]):
feature_norm_all = (mid_feats[:, index] - mean_all) / std_all
feature_norm_fm = (mid_feats[:, index] - mean_fm) / std_fm
_, p1 = at.classifier_wrapper(classifier_all, "knn", feature_norm_all)
_, p2 = at.classifier_wrapper(classifier_fm, "knn", feature_norm_fm)
start = mid_feats.shape[0]
end = mid_feats.shape[0] + len(class_names_all)
mid_term_features[0:mid_feats.shape[0], index] = mid_feats[:, index]
mid_term_features[start:end, index] = p1 + 1e-4
mid_term_features[end::, index] = p2 + 1e-4
mid_feats = mid_term_features # TODO
feature_selected = [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 41,
42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]
mid_feats = mid_feats[feature_selected, :]
mid_feats_norm, mean, std = at.normalize_features([mid_feats.T])
mid_feats_norm = mid_feats_norm[0].T
n_wins = mid_feats.shape[1]
# remove outliers:
dist_all = np.sum(distance.squareform(distance.pdist(mid_feats_norm.T)),
axis=0)
m_dist_all = np.mean(dist_all)
i_non_outliers = np.nonzero(dist_all < 1.2 * m_dist_all)[0]
# TODO: Combine energy threshold for outlier removal:
# EnergyMin = np.min(mt_feats[1,:])
# EnergyMean = np.mean(mt_feats[1,:])
# Thres = (1.5*EnergyMin + 0.5*EnergyMean) / 2.0
# i_non_outliers = np.nonzero(mt_feats[1,:] > Thres)[0]
# print i_non_outliers
mt_feats_norm_or = mid_feats_norm
mid_feats_norm = mid_feats_norm[:, i_non_outliers]
# LDA dimensionality reduction:
if lda_dim > 0:
# extract mid-term features with minimum step:
window_ratio = int(round(mid_window / short_window))
step_ratio = int(round(short_window / short_window))
mt_feats_to_red = []
num_of_features = len(st_feats)
num_of_stats = 2
for index in range(num_of_stats * num_of_features):
mt_feats_to_red.append([])
# for each of the short-term features:
for index in range(num_of_features):
cur_pos = 0
feat_len = len(st_feats[index])
while cur_pos < feat_len:
n1 = cur_pos
n2 = cur_pos + window_ratio
if n2 > feat_len:
n2 = feat_len
short_features = st_feats[index][n1:n2]
mt_feats_to_red[index].append(np.mean(short_features))
mt_feats_to_red[index + num_of_features].\
append(np.std(short_features))
cur_pos += step_ratio
mt_feats_to_red = np.array(mt_feats_to_red)
mt_feats_to_red_2 = np.zeros((mt_feats_to_red.shape[0] +
len(class_names_all) +
len(class_names_fm),
mt_feats_to_red.shape[1]))
limit = mt_feats_to_red.shape[0] + len(class_names_all)
for index in range(mt_feats_to_red.shape[1]):
feature_norm_all = (mt_feats_to_red[:, index] - mean_all) / std_all
feature_norm_fm = (mt_feats_to_red[:, index] - mean_fm) / std_fm
_, p1 = at.classifier_wrapper(classifier_all, "knn",
feature_norm_all)
_, p2 = at.classifier_wrapper(classifier_fm, "knn", feature_norm_fm)
mt_feats_to_red_2[0:mt_feats_to_red.shape[0], index] = \
mt_feats_to_red[:, index]
mt_feats_to_red_2[mt_feats_to_red.shape[0]:limit, index] = p1 + 1e-4
mt_feats_to_red_2[limit::, index] = p2 + 1e-4
mt_feats_to_red = mt_feats_to_red_2
mt_feats_to_red = mt_feats_to_red[feature_selected, :]
mt_feats_to_red, mean, std = at.normalize_features([mt_feats_to_red.T])
mt_feats_to_red = mt_feats_to_red[0].T
labels = np.zeros((mt_feats_to_red.shape[1], ))
lda_step = 1.0
lda_step_ratio = lda_step / short_window
for index in range(labels.shape[0]):
labels[index] = int(index * short_window / lda_step_ratio)
clf = sklearn.discriminant_analysis.\
LinearDiscriminantAnalysis(n_components=lda_dim)
clf.fit(mt_feats_to_red.T, labels)
mid_feats_norm = (clf.transform(mid_feats_norm.T)).T
if n_speakers <= 0:
s_range = range(2, 10)
else:
s_range = [n_speakers]
cluster_labels = []
sil_all = []
cluster_centers = []
for speakers in s_range:
k_means = sklearn.cluster.KMeans(n_clusters=speakers)
k_means.fit(mid_feats_norm.T)
cls = k_means.labels_
means = k_means.cluster_centers_
cluster_labels.append(cls)
cluster_centers.append(means)
sil_1 = []; sil_2 = []
for c in range(speakers):
# for each speaker (i.e. for each extracted cluster)
clust_per_cent = np.nonzero(cls == c)[0].shape[0] / float(len(cls))
if clust_per_cent < 0.020:
sil_1.append(0.0)
sil_2.append(0.0)
else:
# get subset of feature vectors
mt_feats_norm_temp = mid_feats_norm[:, cls == c]
# compute average distance between samples
# that belong to the cluster (a values)
dist = distance.pdist(mt_feats_norm_temp.T)
sil_1.append(np.mean(dist)*clust_per_cent)
sil_temp = []
for c2 in range(speakers):
# compute distances from samples of other clusters
if c2 != c:
clust_per_cent_2 = np.nonzero(cls == c2)[0].shape[0] /\
float(len(cls))
mid_features_temp = mid_feats_norm[:, cls == c2]
dist = distance.cdist(mt_feats_norm_temp.T,
mid_features_temp.T)
sil_temp.append(np.mean(dist)*(clust_per_cent
+ clust_per_cent_2)/2.0)
sil_temp = np.array(sil_temp)
# ... and keep the minimum value (i.e.
# the distance from the "nearest" cluster)
sil_2.append(min(sil_temp))
sil_1 = np.array(sil_1)
sil_2 = np.array(sil_2)
sil = []
for c in range(speakers):
# for each cluster (speaker) compute silhouette
sil.append((sil_2[c] - sil_1[c]) / (max(sil_2[c], sil_1[c]) + 1e-5))
# keep the AVERAGE SILLOUETTE
sil_all.append(np.mean(sil))
imax = int(np.argmax(sil_all))
# optimal number of clusters
num_speakers = s_range[imax]
# generate the final set of cluster labels
# (important: need to retrieve the outlier windows:
# this is achieved by giving them the value of their
# nearest non-outlier window)
cls = np.zeros((n_wins,))
for index in range(n_wins):
j = np.argmin(np.abs(index-i_non_outliers))
cls[index] = cluster_labels[imax][j]
# Post-process method 1: hmm smoothing
for index in range(1):
# hmm training
start_prob, transmat, means, cov = \
train_hmm_compute_statistics(mt_feats_norm_or, cls)
hmm = hmmlearn.hmm.GaussianHMM(start_prob.shape[0], "diag")
hmm.startprob_ = start_prob
hmm.transmat_ = transmat
hmm.means_ = means; hmm.covars_ = cov
cls = hmm.predict(mt_feats_norm_or.T)
# Post-process method 2: median filtering:
cls = scipy.signal.medfilt(cls, 13)
cls = scipy.signal.medfilt(cls, 11)
class_names = ["speaker{0:d}".format(c) for c in range(num_speakers)]
# load ground-truth if available
gt_file = filename.replace('.wav', '.segments')
# if groundtruth exists
if os.path.isfile(gt_file):
seg_start, seg_end, seg_labs = read_segmentation_gt(gt_file)
flags_gt, class_names_gt = segments_to_labels(seg_start, seg_end,
seg_labs, mid_step)
if plot_res:
fig = plt.figure()
if n_speakers > 0:
ax1 = fig.add_subplot(111)
else:
ax1 = fig.add_subplot(211)
ax1.set_yticks(np.array(range(len(class_names))))
ax1.axis((0, duration, -1, len(class_names)))
ax1.set_yticklabels(class_names)
ax1.plot(np.array(range(len(cls))) * mid_step + mid_step / 2.0, cls)
if os.path.isfile(gt_file):
if plot_res:
ax1.plot(np.array(range(len(flags_gt))) *
mid_step + mid_step / 2.0, flags_gt, 'r')
purity_cluster_m, purity_speaker_m = \
evaluate_speaker_diarization(cls, flags_gt)
print("{0:.1f}\t{1:.1f}".format(100 * purity_cluster_m,
100 * purity_speaker_m))
if plot_res:
plt.title("Cluster purity: {0:.1f}% - "
"Speaker purity: {1:.1f}%".format(100 * purity_cluster_m,
100 * purity_speaker_m))
if plot_res:
plt.xlabel("time (seconds)")
if n_speakers <= 0:
plt.subplot(212)
plt.plot(s_range, sil_all)
plt.xlabel("number of clusters")
plt.ylabel("average clustering's sillouette")
plt.show()
return (np.array(range(len(cls))) * mid_step + mid_step / 2.0,cls)
def mid_term_file_classification(input_file, model_name, model_type,
plot_results=False, gt_file=""):
"""
This function performs mid-term classification of an audio stream.
Towards this end, supervised knowledge is used,
i.e. a pre-trained classifier.
ARGUMENTS:
- input_file: path of the input WAV file
- model_name: name of the classification model
- model_type: svm or knn depending on the classifier type
- plot_results: True if results are to be plotted using
matplotlib along with a set of statistics
RETURNS:
- segs: a sequence of segment's endpoints: segs[i] is the
endpoint of the i-th segment (in seconds)
- classes: a sequence of class flags: class[i] is the
class ID of the i-th segment
"""
labels = []
accuracy = 0.0
class_names = []
cm = np.array([])
if not os.path.isfile(model_name):
print("mtFileClassificationError: input model_type not found!")
return labels, class_names, accuracy, cm
# Load classifier:
if model_type == "knn":
classifier, mean, std, class_names, mt_win, mid_step, st_win, \
st_step, compute_beat = at.load_model_knn(model_name)
else:
classifier, mean, std, class_names, mt_win, mid_step, st_win, \
st_step, compute_beat = at.load_model(model_name)
if compute_beat:
print("Model " + model_name + " contains long-term music features "
"(beat etc) and cannot be used in "
"segmentation")
return labels, class_names, accuracy, cm
# load input file
sampling_rate, signal = audioBasicIO.read_audio_file(input_file)
# could not read file
if sampling_rate == 0:
return labels, class_names, accuracy, cm
# convert stereo (if) to mono
signal = audioBasicIO.stereo_to_mono(signal)
# mid-term feature extraction:
mt_feats, _, _ = \
mtf.mid_feature_extraction(signal, sampling_rate,
mt_win * sampling_rate,
mid_step * sampling_rate,
round(sampling_rate * st_win),
round(sampling_rate * st_step))
posterior_matrix = []
# for each feature vector (i.e. for each fix-sized segment):
for col_index in range(mt_feats.shape[1]):
# normalize current feature v
feature_vector = (mt_feats[:, col_index] - mean) / std
# classify vector:
label_predicted, posterior = \
at.classifier_wrapper(classifier, model_type, feature_vector)
labels.append(label_predicted)
# update probability matrix
posterior_matrix.append(np.max(posterior))
labels = np.array(labels)
# convert fix-sized flags to segments and classes
segs, classes = labels_to_segments(labels, mid_step)
segs[-1] = len(signal) / float(sampling_rate)
# Load grount-truth:
labels_gt, class_names_gt, accuracy, cm = \
load_ground_truth(gt_file, labels, class_names, mid_step, plot_results)
return labels, class_names, accuracy, cm
