def download_TransformedFeature_histograms(
featureName, NNtype, numVisible, numHidden, numFolders,
numFilesPerFolder, batchSize, learningRate, learningRateBoostFactor,
corruptionLevel, timeStacking, frequencyStandardisation):
# Load weights, biases, feature offset and feature scaling
weights, biases = get_NN_WeightsAndBiases(NNtype, featureName, batchSize,
learningRate,
learningRateBoostFactor,
corruptionLevel, timeStacking,
frequencyStandardisation,
numVisible, numHidden)
featureOffset, featureScaling = get_NN_featureOffsetAndScaling(
featureName, numFolders, numFilesPerFolder, timeStacking, numVisible,
frequencyStandardisation)
# Get histogram of transformed features
hist = transformedFeatureFrequenciesHistogram(featureName,
weights,
biases,
featureOffset,
featureScaling,
timeStacking,
numFolders,
numFilesPerFolder,
numBins=100)
# Save histogram
resultsPath = transferPath + 'feature histograms/'
createPath(resultsPath)
pickle.dump(hist, open(resultsPath + featureName +
'_crpt_%i_nin_%i_nhdn_%i_basz_%i_lnrt_%0.2f' \
% (corruptionLevel, numVisible, numHidden, batchSize, learningRate) +
'_transformed_features_frequency_histograms.pkl', 'wb'))
def download_Feature_histograms(
featureNames=[
'chroma', 'constantQspec', 'mfcc', 'logfreqspec', 'CENS_dsf2_wl41',
'logfreqspec-dA-256v-12h-FENS_dsf2_dswl41'
],
transformedFeatureNames=['logfreqspec-dA-256v-12h-FENS_dsf2_dswl41']):
resultsPath = transferPath + 'feature histograms/'
createPath(resultsPath)
for featureName in featureNames:
featureHist = featureFrequenciesHistogram(featureName, 20, 5)
pickle.dump(
featureHist,
open(resultsPath + featureName + '_frequency_histograms.pkl',
'wb'))
for featureName in transformedFeatureNames:
featureHist = featureFrequenciesHistogram(featureName,
20,
5,
featureFileType='.pkl')
pickle.dump(
featureHist,
open(
resultsPath + featureName +
'_transformed_features_frequency_histograms.pkl', 'wb'))
def moveResultsFiles(subFolder):
NCDdest = NCDpath + subFolder + '/'
runHistDest = runHistoryPath + subFolder + '/'
createPath(NCDdest)
createPath(runHistDest)
moveFiles(NCDpath, NCDdest, '.pkl.res')
moveFiles(runHistoryPath, runHistDest, '.pkl')
return NCDdest
def download_NN_training_results(NNtype='dA'):
trainingPath = '%straining_results/%s/' % (NNpath, NNtype)
resultsFolders = getFolderNames(trainingPath, orderAlphabetically=True)
outputPath = transferPath + 'NN training results/'
createPath(outputPath)
for folder in resultsFolders:
resultsPath = trainingPath + folder + '/'
resultsFn = resultsPath + 'training_records.pkl'
if os.path.exists(resultsFn):
shutil.copyfile(resultsFn, outputPath + folder + '.pkl')
runDict = {'featureName': featureName,
'method': CRPmethod,
'numFilesPerFolder': numFilesPerFolder,
'networkType': NNtype}
for key in setting.keys():
runDict[key] = setting[key]
runTime = str(datetime.now()).replace(':', '-')
pickle.dump(runDict, open(runHistoryPath + runTime + '.pkl', 'wb'))
# Convert NCD files into a dataframe
dfNCDs = convertNCDfiles(dataFrameFileName = runTime)
# Create subfolders and move results files into them
NCDdest = NCDpath + subFolder + '/'
runHistDest = runHistoryPath + subFolder + '/'
createPath(NCDdest)
createPath(runHistDest)
moveFiles(NCDpath, NCDdest, '.pkl.res')
moveFiles(runHistoryPath, runHistDest, '.pkl')
# Get the overall MAP of the run and add to the setting
MAPresult = getDataFrameMAPresult(dfNCDs)
if MAPresult is not None:
print 'Mean Average Precision: %0.3f\n' % MAPresult
else:
print 'No MAP result found!'
setting['Mean Average Precision'] = MAPresult
# Increment the number of runs
numRuns += len(nextSettings)
from create_ncds_val_test import createNCDfiles
from ncd_processing import convertNCDfiles
from getResults2 import getDataFrameMAPresult
from NNs import get_NN_NCD_params
from MAPhelpers import cleanCRPfolder, cleanNCDfolder
# Test some settings on the validation set, i.e. all the remaining files in the
# training folder. The results should be slightly better than would get on a test
# set since the pieces are the same and only the performances differ.
numProcesses = 10
settingsFileName = '/u7.swansea/s11/abpg162/project/run_settings/Run Settings Template.csv'
settingsDicts = loadRunSettingsList(settingsFileName)
# Create folder for results
createPath(NCDpath + 'validation runs')
# Initialise
processPool = Pool(numProcesses)
for settingsDict in settingsDicts:
resultsFn = '/u7.swansea/s11/abpg162/project/results_files/validation/' + settingsDict[
'Run Name'] + '.pkl'
if not os.path.exists(resultsFn):
cleanCRPfolder()
cleanNCDfolder()
startDateTime = datetime.now()
# load weights etc. if this is for a neural net run
NNtype = None
learningRate = 0.05
learningRateBoostFactor = 1.5
batchSize = 40
numVisibles = [12]
numHiddens = [12]
corruptionLevels = [0]
timeStacking = 1
freqStd = False
stopRunningAt = datetime(2015, 9, 23, 23)
###############################################################################
# Create folder for results
createPath(NCDpath + subFolder)
# Create a dictionary of settings to optimise over
settingsDict = {
'Dimension': CRPdimensions,
'DownSample Factor': downSampleFactors,
'Neighbourhood Size': CRPneighbourhoodSizes,
'Sequence Length': sequenceLengths,
'Time Delay': CRPtimeDelays
}
if NNtype is not None:
settingsDict['dA Num Hidden Units'] = numHiddens
settingsDict['dA Num Visible Units'] = numVisibles
settingsDict['dA Corruption Level'] = corruptionLevels
NNtype = 'dA'
learningRate = 0.05
learningRateBoostFactor = 1.5
batchSize = 40
numVisibles = [48]
numHiddens = [12]
corruptionLevels = [0, 10, 20, 30, 40, 50, 60, 70]
timeStacking = 1
freqStd = True
stopRunningAt = datetime(2015, 9, 19, 1)
###############################################################################
# Create folder for results
createPath(NCDpath + subFolder)
createPath(runHistoryPath + subFolder)
# Create a dictionary of settings to optimise over
settingsDict = {
'Dimension': CRPdimensions,
'DownSample Factor': downSampleFactors,
'Neighbourhood Size': CRPneighbourhoodSizes,
'Sequence Length': sequenceLengths,
'Time Delay': CRPtimeDelays,
'FENS Window Length': FENSwindowLengths,
'FENS Downsampling': FENSdownsamplings,
'FENS Normalisation Threshold': FENSnormalisationThresholds,
'FENS Quantisation Step Base': FENSquantisationStepBases,
'FENS Quantisation Step Power': FENSquantisationStepPowers,
'FENS Quantisation Weight 1': FENSquantisationWeights1,
from create_ncds_val_test import createNCDfiles
from ncd_processing import convertNCDfiles
from getResults2 import getDataFrameMAPresult
from NNs import get_NN_NCD_params
from MAPhelpers import cleanCRPfolder, cleanNCDfolder
# Test some settings on the validation set, i.e. all the remaining files in the
# training folder. The results should be slightly better than would get on a test
# set since the pieces are the same and only the performances differ.
numProcesses = 10
settingsFileName = '/u7.swansea/s11/abpg162/project/run_settings/Run Settings - Test.csv'
settingsDicts = loadRunSettingsList(settingsFileName)
# Create folder for results
createPath(NCDpath + 'test runs')
# Initialise
processPool = Pool(numProcesses)
for settingsDict in settingsDicts:
resultsFn = '/u7.swansea/s11/abpg162/project/results_files/test/' + settingsDict[
'Run Name'] + '.pkl'
if not os.path.exists(resultsFn):
cleanCRPfolder()
cleanNCDfolder()
startDateTime = datetime.now()
# load weights etc. if this is for a neural net run