def cutSpectrumMat(audioRecordingId,
inputSpectrumFolder,
outputSpectrumSlicesFolder,
sizeSquareImages,
additionalText='_spectrum.mat',
structName='new_spectrum'):
spectrumFilePath = "".join(
[inputSpectrumFolder, audioRecordingId, additionalText])
if (os.path.exists(spectrumFilePath)):
spectrum_data = (loader.loadMatFiles(spectrumFilePath))[structName]
sizeSpectrum = np.shape(spectrum_data)
rst = sizeSpectrum[-1] % sizeSquareImages
if (rst != 0):
cols2add = sizeSquareImages - rst
minimum = spectrum_data.min()
spectrum_data = np.hstack([
np.ones([sizeSpectrum[0], cols2add]) * minimum, spectrum_data
])
numSlices = int((np.shape(spectrum_data)[1]) / sizeSquareImages)
print("CREATING SLICES: nImages", numSlices, " for file: ",
audioRecordingId)
for sliceIndex in range(0, numSlices):
indexColumnStart = (sliceIndex) * sizeSquareImages
indexColumnEnd = ((sliceIndex + 1) * sizeSquareImages)
spectrum_data_slice = spectrum_data[
0:sizeSquareImages, indexColumnStart:indexColumnEnd]
imgOutputPath = "".join([
outputSpectrumSlicesFolder, "/", audioRecordingId, "_",
str(sliceIndex), '.mat'
])
saver.saveMatrixAsMat(spectrum_data_slice,
imgOutputPath,
nameMatrixMatlab=structName)
def normPercentileZscorePerUser(input_file_path, typeOfNormalization, newSupValue, newInfValue, outputPath,text2add, percentile):
functionals_data = (loader.loadMatFiles(input_file_path))['fixed_spectrum_matrix']
output_spectrum_name = (input_file_path.split(settings.operative_system)[-1].split('.mat')[0]) + text2add
#functionals_data_modi = np.reshape(functionals_data, [-1])
# plt.figure()
# plt.hist(functionals_data_modi, bins = 200)
#plt.show()
#nameFig = "".join([outputPath, (input_file_path.split('/')[-1].split('.mat')[0]), '_distribution.png'])
#plt.savefig(nameFig)
#zscore
if (typeOfNormalization == 7): # total of the whole audio file
meanValue = functionals_data.mean()
stdValue = functionals_data.std()
else:
print("Error in type of normalization")
return -1
zscore_data = getZscoreData(functionals_data, meanValue, stdValue) #no debería importar
#zscore_data_modi = np.reshape(zscore_data, [-1])
# plt.figure()
# plt.hist(zscore_data_modi, bins = 200)
#plt.show()
#percentile
percentileValue = np.percentile(zscore_data, percentile)
zscore_data[zscore_data < percentileValue] = percentileValue
# zscore_data_modiP = np.reshape(zscore_data, [-1])
# plt.figure()
# plt.hist(zscore_data_modiP, bins=200)
# plt.show()
#normalization
if (typeOfNormalization == 7): # total of the whole audio file
minimumValue = zscore_data.min()
maximumValue = zscore_data.max()
else:
print("Error in type of normalization")
return -1
new_spectrum = getNormalizedData(zscore_data, minimumValue, maximumValue, newSupValue=newSupValue, newInfValue=newInfValue)
# img = Image.fromarray(new_spectrum[:, :].astype('uint8'), 'L')
# imgOutputPath = '/home/cris/PycharmProjects/MasterProject/data/RESULTS/otrasPruebas/zscore.png'
# img.save(imgOutputPath, "PNG")
# plt.figure()
# plt.imshow(new_spectrum)
# plt.show()
#new_spectrum_modi = np.reshape(new_spectrum, [-1])
# plt.figure()
# plt.hist(new_spectrum_modi, bins = 200)
#plt.show()
savePath = "".join([outputPath, output_spectrum_name])
saver.saveMatrixAsMat(new_spectrum, savePath, 'fixed_spectrum_matrix')
return meanValue, stdValue, minimumValue, maximumValue
def create_imgs_from_spectrums_parallel(mat_file_name,
input_mat_path,
output_imgs_folder,
sizeSquareImages,
codeChannels=32):
# cration of images with(out) labels
input_mat_path = "".join([input_mat_path, mat_file_name])
fileName = mat_file_name.split(".mat")[0]
print("Processing image: " + mat_file_name)
if (os.path.exists(input_mat_path) and ".mat" in input_mat_path):
spectrum_data = (
loader.loadMatFiles(input_mat_path))['fixed_spectrum_matrix']
out_path = "".join([output_imgs_folder, fileName, ".png"])
img, _, _ = getImageByCodeChannel(codeChannels, spectrum_data,
sizeSquareImages, mat_file_name)
saver.save_img(img, out_path)
def create_imgs_from_spectrums(mat_folder,
output_imgs_folder,
codeChannels=32):
# cration of images with(out) labels
listSpectrumFiles = sorted(os.listdir(mat_folder))
for spectrum_file in listSpectrumFiles:
fileName = spectrum_file.split(".")
if (".mat" in spectrum_file):
print("Processing image: " + spectrum_file)
input_file_path = "".join([mat_folder, spectrum_file])
spectrum_data = (
loader.loadMatFiles(input_file_path))['fixed_spectrum_matrix']
sizeSpectrum = np.shape(spectrum_data)
out_path = "".join([output_imgs_folder, fileName[0], ".png"])
img, _, _ = getImageByCodeChannel(codeChannels, spectrum_data,
sizeSpectrum[0], spectrum_file)
saver.save_img(img, out_path)
def normalizationPerUser(input_file_path,typeOfNormalization,newSupValue,newInfValue,outputPath, text2add='_norm0255.mat', name = 'fixed_spectrum_matrix', matInternalName = 'fixed_spectrum_matrix'):
functionals_data = (loader.loadMatFiles(input_file_path))[name]
if (typeOfNormalization == 1): # total of the whole audio file
minimumValue = functionals_data.min()
maximumValue = functionals_data.max()
elif (typeOfNormalization == 2): # min/max per frequency (160, 224 or 672 depending on number of FFT points)
minimumValue = functionals_data.min(1)
maximumValue = functionals_data.max(1)
minimumValue = np.reshape(minimumValue, [-1, 1])
maximumValue = np.reshape(maximumValue, [-1, 1])
else:
print("Error in type of normalization")
return -1
new_spectrum = getNormalizedData(functionals_data,minimumValue,maximumValue,newSupValue,newInfValue)
#save data
output_spectrum_name = (input_file_path.split('/')[-1].split('.mat')[0]) + text2add
savePath = "".join([outputPath, output_spectrum_name])
saver.saveMatrixAsMat(new_spectrum, savePath, matInternalName)
return minimumValue,maximumValue
def normPercentilePerUser(input_file_path,typeOfNormalization,newSupValue,newInfValue,outputPath, text2add='_norm0255.mat', percentile = 25):
functionals_data = (loader.loadMatFiles(input_file_path))['fixed_spectrum_matrix']
minimumValue=[]
maximumValue=[]
percentileValue = np.percentile(functionals_data, percentile)
functionals_data[functionals_data < percentileValue] = percentileValue
if (typeOfNormalization == 5): # total of the whole audio file
minimumValue = functionals_data.min()
maximumValue = functionals_data.max()
else:
print("Error in type of normalization")
return -1
output_spectrum_name = (input_file_path.split('/')[-1].split('.mat')[0]) + text2add
new_spectrum = getNormalizedData(functionals_data, minimumValue, maximumValue, newSupValue, newInfValue)
#img = Image.fromarray(new_spectrum[:, :].astype('uint8'), 'L')
# imgOutputPath = '/home/cris/PycharmProjects/MasterProject/data/RESULTS/otrasPruebas/norm.png'
# img.save(imgOutputPath, "PNG")
savePath = "".join([outputPath, output_spectrum_name])
saver.saveMatrixAsMat(new_spectrum, savePath, 'fixed_spectrum_matrix')
def zscorePerUser(input_file_path,typeOfNormalization,newSupValue,newInfValue,outputPath,zscoreThreshold=None, text2add='_zscore0255.mat', name = 'fixed_spectrum_matrix', newName ='fixed_spectrum_matrix' ):
functionals_data = (loader.loadMatFiles(input_file_path))[name]
meanValue = []
stdValue = []
minimumValue = []
maximumValue = []
if (typeOfNormalization == 3): # total of the whole audio file
meanValue = functionals_data.mean()
stdValue = functionals_data.std()
elif (typeOfNormalization == 4): # min/max per frequency (224 or 672 depending on number of FFT points)
meanValue = functionals_data.mean(1)
stdValue = functionals_data.std(1)
meanValue = np.reshape(meanValue, [-1, 1])
stdValue = np.reshape(stdValue, [-1, 1])
else:
print("Error in type of normalization")
return -1
output_spectrum_name = (input_file_path.split(settings.operative_system)[-1].split('.mat')[0]) + text2add
new_spectrum,minimumValue,maximumValue = getZscoreNormData(functionals_data, meanValue, stdValue, minimumValue, maximumValue, newSupValue, newInfValue, zscoreThreshold)
savePath = "".join([outputPath, output_spectrum_name])
saver.saveMatrixAsMat(new_spectrum, savePath, newName)
return meanValue, stdValue, minimumValue, maximumValue
def spectrumNormalization(input_path_FFTspectrums, outputPath,listTrainingFilesNames,listValidationFilesNames,typeOfNormalization= 1,
perUserOrAllUsers="perUser", newInfValue=0, newSupValue=255, obtainTest=False, text2add="_norm0255",
saveMinMaxPickle=False, zscoreThreshold=None, percentileNormalizationType4=25):
listSpectrumFiles = os.listdir(input_path_FFTspectrums)
trainingData = []
trainingNames = []
validationData = []
validationNames = []
testData = []
testNames = []
minimumValue = [] #mean or min
maximumValue = []# std or max
meanValue = []
stdValue = []
for spectrum_file in listSpectrumFiles:
nameSpectrum = spectrum_file.split(".")
output_spectrum_name = "".join([nameSpectrum[0],text2add,'.mat'])
if(".mat" in spectrum_file):
input_file_path = "".join([input_path_FFTspectrums, spectrum_file])
#normalization
if(perUserOrAllUsers=="perUser"): # if normalization is per user, we obtain the normalized value of all the users(train,dev&test)
#create and save normalized data
if(typeOfNormalization==1 or typeOfNormalization==2):
normalizationPerUser(typeOfNormalization=typeOfNormalization, input_file_path=input_file_path, newSupValue=newSupValue,newInfValue= newInfValue, outputPath=outputPath,
text2add="".join([text2add,'.mat']))
elif(typeOfNormalization==3 or typeOfNormalization==4):
zscorePerUser(input_file_path, typeOfNormalization, newSupValue, newInfValue, outputPath,
zscoreThreshold=zscoreThreshold, text2add=text2add)
elif(typeOfNormalization==5):
normPercentilePerUser(input_file_path, typeOfNormalization, newSupValue, newInfValue, outputPath,
text2add="".join([text2add, '.mat']), percentile=percentileNormalizationType4)
elif(typeOfNormalization==7):
normPercentileZscorePerUser(input_file_path, typeOfNormalization, newSupValue, newInfValue,
outputPath, text2add, percentile=percentileNormalizationType4)
elif(perUserOrAllUsers=="allUsers"):
#create array with all the normalized data but not save it until the end
functionals_data = (loader.loadMatFiles(input_file_path))['fixed_spectrum_matrix']
sizeSpectrum = np.shape(functionals_data)
recordName = spectrum_file.split("_spectrum.mat")
if(recordName[0] in listTrainingFilesNames):
trainingData,trainingNames = addData2Lists(functionals_data, trainingData, output_spectrum_name, trainingNames)
elif(recordName[0] in listValidationFilesNames):
validationData,validationNames = addData2Lists(functionals_data, validationData, output_spectrum_name, validationNames)
elif(("test" in output_spectrum_name)):
if(obtainTest):
testData,testNames = addData2Lists(functionals_data, testData, output_spectrum_name, testNames)
else:
print("file not in training or validation: ",recordName[0])
else:
print('Not valid option for normalization, choose: perUser or allUsers')
return -1
if (perUserOrAllUsers=="allUsers"): #FALTA IMPLEMENTAR NORMALIZATION 5 Y 6 PARA TODOS LOS USUARIO, POR AHORA SÓLO PER USER
#SCALING
if(typeOfNormalization==1 or typeOfNormalization==2):
if(typeOfNormalization==1):
minimumValue = trainingData.min()
maximumValue = trainingData.max()
elif(typeOfNormalization==2):
minimumValue = trainingData.min(1)
maximumValue = trainingData.max(1)
minimumValue = np.reshape(minimumValue, [-1, 1])
maximumValue = np.reshape(maximumValue, [-1, 1])
trainingData = np.array(trainingData)
validationData = np.array(validationData)
trainingDataNorm = getNormalizedData(trainingData, minimumValue, maximumValue, newSupValue, newInfValue)
validationDataNorm = getNormalizedData(validationData, minimumValue, maximumValue, newSupValue, newInfValue)
if (obtainTest):
testData = np.array(testData)
testDataNorm = getNormalizedData(testData, minimumValue, maximumValue, newSupValue, newInfValue)
#ZSCORE + SCALING
elif(typeOfNormalization==3 or typeOfNormalization==4):
if (typeOfNormalization == 3): # total of the whole audio file
meanValue = trainingData.mean()
stdValue = trainingData.std()
elif (typeOfNormalization == 4): # min/max per frequency (224 or 672 depending on number of FFT points)
meanValue = trainingData.mean(1)
stdValue = trainingData.std(1)
meanValue = np.reshape(meanValue, [-1, 1])
stdValue = np.reshape(stdValue, [-1, 1])
trainingData = np.array(trainingData)
validationData = np.array(validationData)
trainingDataNorm,minimumValue,maximumValue = getZscoreNormData(trainingData, meanValue, stdValue, minimumValue, maximumValue, newSupValue, newInfValue, zscoreThreshold=zscoreThreshold)
validationDataNorm,_,_ = getZscoreNormData(validationData, meanValue, stdValue, minimumValue, maximumValue, newSupValue, newInfValue, zscoreThreshold=zscoreThreshold)
if (obtainTest):
testData = np.array(testData)
testDataNorm,_,_ = getZscoreNormData(testData, meanValue, stdValue, minimumValue, maximumValue, newSupValue, newInfValue, zscoreThreshold=zscoreThreshold)
else:
print("Error in type of normalization")
return -1
if(not trainingNames == []):
saveNormalizedFiles4AllUsers(trainingNames, sizeSpectrum, trainingDataNorm, outputPath)
if(not validationNames == []):
saveNormalizedFiles4AllUsers(validationNames, sizeSpectrum, validationDataNorm, outputPath)
if (obtainTest):
saveNormalizedFiles4AllUsers(testNames, sizeSpectrum, testDataNorm, outputPath)
if(saveMinMaxPickle):
if(typeOfNormalization==1 or typeOfNormalization==2):
matrix = np.array([minimumValue,maximumValue])
elif(typeOfNormalization==3 or typeOfNormalization==4):
matrix = np.array([meanValue, stdValue,minimumValue,maximumValue])
saver.savePicklefromMatrix(outputPath, matrix, ('matrixTrainingAllUsersCase'+str(typeOfNormalization)))
return meanValue, stdValue,minimumValue,maximumValue