def learnRegressorFromDataset(regressorTypeToLearn, transitionId, YSet,
confsList, SubjectSystem):
"""
Calculates the regressor that CV found to be the best using all the data from training
Requries SubjectSystem to decide how to generate bitmamps from configuration id.
"""
RegressionType = regressorTypeToLearn[0]
UseSquares = regressorTypeToLearn[1]
AlphaIndex = regressorTypeToLearn[2]
SingleYList = []
[SingleYList.extend(YBag) for YBag in YSet]
YLocalScaler = StandardScaler()
YLocalScaler.fit([[target] for target in SingleYList])
SingleYScaledList = YLocalScaler.transform([[aY] for aY in SingleYList])
if SubjectSystem == MLConstants.x264Name:
XBitmaps = getXBitmapsForRegression(confsList, YSet, UseSquares)
else:
# autonomoose
XBitmaps = getXBitmapsForRegression(
confsList, YSet, UseSquares,
generateBitsetForOneConfigurationAutonomoose)
if RegressionType == MLConstants.simpleRegression:
Regressor = LinearRegression()
elif RegressionType == MLConstants.ridgeRegression:
Regressor = Ridge(alpha=alphaValues[AlphaIndex])
elif RegressionType == MLConstants.lassoRegression:
Regressor = Lasso(alpha=alphaValues[AlphaIndex])
Regressor.fit(XBitmaps, SingleYScaledList)
YPredicted = YLocalScaler.inverse_transform(Regressor.predict(XBitmaps))
YOriginalArray = np.array(SingleYList)
YOriginalArray.resize((len(SingleYList), 1))
# Standarize as Lasso returns array of different shape than linear and ridge regression.
if RegressionType == MLConstants.lassoRegression:
YPredicted.resize((len(SingleYList), 1))
MAPEYTrain, MAPEStdTrain = mean_absolute_error_and_stdev_eff(
YOriginalArray, YPredicted)
return Regressor, UseSquares, RegressionType == MLConstants.lassoRegression, YLocalScaler, YLocalScaler.mean_, np.sqrt(
YLocalScaler.var_), MAPEYTrain, MAPEStdTrain
def analyzeOverallExecutionTimesX264(regressorsArray, testConfigurationsList,
transitionToRegressorMapping,
transitionToConfArrayTimeTaken):
"""
Calculate time taken for each execution in confs that are part of test set.
"""
listActualTimes = []
listPredictedTimes = []
for aConfId, offsetIndex in zip(testConfigurationsList,
range(0, len(testConfigurationsList))):
for repetitionId in range(1, 11):
timeTameknDict = sumTimeTakenPerTransitionFromConfigurationAndRep(
aConfId, repetitionId)
timeTakenByTraceAddition = sum([
timeTameknDict[x][MLConstants.tupleTimeOffset]
for x in timeTameknDict.keys()
])
predictedTimeTaken = 0
for foundTransitionId in timeTameknDict.keys():
if foundTransitionId in transitionToRegressorMapping.keys():
predictedTimeTaken = predictedTimeTaken + (
transitionToConfArrayTimeTaken[foundTransitionId]
[offsetIndex] * timeTameknDict[foundTransitionId][
MLConstants.tupleCountOffset])
print("{0},{1},{2}".format(aConfId, timeTakenByTraceAddition,
predictedTimeTaken[0]))
listActualTimes.append(timeTakenByTraceAddition)
listPredictedTimes.append(predictedTimeTaken)
npActualTimes = np.array([np.array([x]) for x in listActualTimes])
npPredictedTimes = np.array(listPredictedTimes)
meanMAE, stdMAE = mean_absolute_error_and_stdev_eff(
npActualTimes, npPredictedTimes)
print("Mean_MAE,MAE_STDEV")
print("{0},{1}".format(meanMAE, stdMAE))
def calculateTestErrors(Regressor, RegressorUseSquares, RegressorIsLasso,
YLocalScaler, transitionId, YSet, testOrderedConfs,
SubjectSystem):
"""
given a regressor function, calculates the error of such regressor on the test sets.
Computes MAE_Test and MAE_Test std. dev across products.
"""
SingleYList = []
[SingleYList.extend(YBag) for YBag in YSet]
YTestOriginalArray = np.array(SingleYList)
YTestOriginalArray.resize((len(SingleYList), 1))
if SubjectSystem == MLConstants.x264Name:
XBitmaps = getXBitmapsForRegression(testOrderedConfs, YSet,
RegressorUseSquares)
else:
# autonomoose
XBitmaps = getXBitmapsForRegression(
testOrderedConfs, YSet, RegressorUseSquares,
generateBitsetForOneConfigurationAutonomoose)
YTestPredicted = YLocalScaler.inverse_transform(
Regressor.predict(XBitmaps))
# print("Shape Y_TestOriginal {0}".format( YTestOriginalArray.shape))
if RegressorIsLasso:
YTestPredicted.resize((len(SingleYList), 1))
# print("Shape Y_TestPredicted {0}".format( YTestPredicted.shape))
MAPEYTest, MAEYStdDevTest = mean_absolute_error_and_stdev_eff(
YTestOriginalArray, YTestPredicted)
return np.mean(YTestOriginalArray), np.std(
YTestOriginalArray), MAPEYTest, MAEYStdDevTest
def analyzeOverallExecutionTimesAutonomoose(regressorsArray,
testConfigurationsList,
transitionToRegressorMapping,
transitionToConfArrayTimeTaken):
"""
Autonomoose all traces are in a single file.
Calculate actual execution time for trace of autonomoose, as well as predicted execution time.
"""
listActualTimes = []
listPredictedTimes = []
allTraces = loadObjectFromPickle(getSingleFilenameWithAllTraces())
# print (len(testConfigurationsList))
for aConfId, offsetIndex in zip(testConfigurationsList,
range(0, len(testConfigurationsList))):
#print("Computing for Configuration {0} with offset Index = {1}".format(aConfId, offsetIndex))
for repetitionId in range(0, 10):
CurrentExecutionTrace = allTraces[aConfId][repetitionId]
actualExecutionTime = getOverallRealTimeForASingleTraceAutonomoose(
CurrentExecutionTrace, aConfId)
transitionsCounts = CurrentExecutionTrace.getPerTransitionCounts()
#print(transitionsCounts)
predictedTimeTaken = 0.0
for aTransitionId in transitionsCounts.keys():
predictedTimeTaken = predictedTimeTaken + transitionsCounts[
aTransitionId] * transitionToConfArrayTimeTaken[
aTransitionId][offsetIndex]
if not (type(predictedTimeTaken) == float):
print("{0},{1},{2}".format(aConfId, actualExecutionTime,
predictedTimeTaken[0]))
else:
print("{0},{1},{2}".format(aConfId, actualExecutionTime,
predictedTimeTaken))
listActualTimes.append(actualExecutionTime)
listPredictedTimes.append(predictedTimeTaken)
npActualTimes = np.array([np.array([x]) for x in listActualTimes])
npPredictedTimes = np.array(listPredictedTimes)
meanMAE, stdMAE = mean_absolute_error_and_stdev_eff(
npActualTimes, npPredictedTimes)
print("Mean_MAE,MAE_STDEV")
if meanMAE.shape == (1, ):
print("{0},{1}".format(meanMAE[0], stdMAE[0]))
else:
print("{0},{1}".format(meanMAE, stdMAE))
def analyzeAutonomooseFSE(trainConfigurationList, testConfigurationsList):
"""
Analyze execution time of autonomooose
Parameters
-----------
traceExecutionTimesSummaries : Dictionary
Maps tuples (confId, repetitionId) to execution time (microseconds)
trainConfigurationList : List of integers
testConfigurationsList : List of integers
Notes
------
1. Open all tracess files.
BooleanOptions = [("BEHAVIOR", 4 ), \
("OCCUPANCY", 2),
("WAYPOINTS", 1)]
Behavior Planner
Occupancy or Mockupancy Planner
Waypoints Collection
Dyn. Object Tracking
Dyn. Car Tracking.
Dyn. Person Tracking
TODO - REFACTOR to join it into analyzeX264FSE -- strategy pattern.
"""
allTraces = loadObjectFromPickle(getSingleFilenameWithAllTraces())
vmAutonomoose = generateFullAutonomooseVariabilityModel()
InfModelAutonomoose = InfluenceModels.InfluenceModel(vmAutonomoose)
TmpMLSettings = MLSettings.MLSettings()
TmpMLSettings.useBackward = True
lstFSETrainConfigurationListing = transformToFSEFormat(trainConfigurationList, vmAutonomoose, \
ConfigurationIdToBitsetTransformer=generateBitsetForOneConfigurationAutonomoose, \
BitsetToFseTransformer=transformSingleConfigurationToAutonomooseFSE)
setNFPValuesForConfigurationListAutonomoose(
lstFSETrainConfigurationListing, trainConfigurationList, allTraces)
tmpSubsetSelection = FeatureSubsetSelection.FeatureSubsetSelection(
InfModelAutonomoose, TmpMLSettings)
tmpSubsetSelection.setLearningSet(lstFSETrainConfigurationListing)
tmpSubsetSelection.setValidationSet(
lstFSETrainConfigurationListing
) # Following FSE paper, Learning set is resued as -- 'validation set' --
tmpSubsetSelection.learn()
showInfluenceModel = False
if showInfluenceModel:
printInfluenceModel(tmpSubsetSelection)
lstFSETestConfigurationListing = transformToFSEFormat(testConfigurationsList, vmAutonomoose, \
ConfigurationIdToBitsetTransformer=generateBitsetForOneConfigurationAutonomoose, \
BitsetToFseTransformer=transformSingleConfigurationToAutonomooseFSE)
#
#
#
# CALCULATING NMAE WITHOUT USING AVERAGING.
averagingTestSet = False
if averagingTestSet == True:
setNFPValuesForConfigurationListAutonomoose(
lstFSETestConfigurationListing, testConfigurationsList, allTraces)
lstMeasuredValues = [
x.getNfpValue() for x in lstFSETestConfigurationListing
]
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
else:
lstMeasuredValues = []
for aConfId in testConfigurationsList:
for repetitionId in range(0, 10):
executedTimes = getOverallRealTimeForASingleTraceAutonomoose(
allTraces[aConfId][repetitionId], aConfId)
lstMeasuredValues.append(executedTimes)
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
tmpLstEstimatedValues = []
for estimatedVal in lstEstimatedValues:
for i in range(0, 10):
tmpLstEstimatedValues.append(estimatedVal)
lstEstimatedValues = tmpLstEstimatedValues
lstMeasuredValuesNp = np.array(lstMeasuredValues)
lstEstimatedValuesNp = np.array(lstEstimatedValues)
MAETestMean, MAETestStd = mean_absolute_error_and_stdev_eff(
lstMeasuredValuesNp, lstEstimatedValuesNp)
MEANTestVal = np.mean(lstMeasuredValuesNp)
NormalizedMae = 100 * (MAETestMean / MEANTestVal)
print("MAE_TEST_MEAN, MAE_TEST_STD, MEAN_TEST, NOMRALIZED_MAE (%) ")
print("{0},\t {1},\t {2},\t {3}".format(MAETestMean, MAETestStd,
MEANTestVal, NormalizedMae))
def analyzeX264FSE(trainConfigurationList, testConfigurationsList,
traceExecutionTimesSummaries):
"""
Analyze x264 executions using FSE paper.
Parameters
-----------
traceExecutionTimesSummaries : Dictionary
Maps tuples (confId, repetitionId) to execution time (microseconds)
trainConfigurationList : List of integers
testConfigurationsList : List of integers
"""
vmX264 = generateFullX264VariabilityModel()
InfModelX264 = InfluenceModels.InfluenceModel(vmX264)
TmpMLSettings = MLSettings.MLSettings()
TmpMLSettings.useBackward = True
lstFSETrainConfigurationListing = transformToFSEFormat(
trainConfigurationList, vmX264)
# Set average values for each configuration
setNFPValuesForConfigurationList(lstFSETrainConfigurationListing,
trainConfigurationList,
traceExecutionTimesSummaries)
# print (lstFSETrainConfigurationListing[0].getNfpValue())
# for repId in range(X264_RANGE_START, X264_RANGE_END):
# print (traceExecutionTimesSummaries[(trainConfigurationList[0], repId)])
# #print(lstFSETrainConfigurationListing)
tmpSubsetSelection = FeatureSubsetSelection.FeatureSubsetSelection(
InfModelX264, TmpMLSettings)
tmpSubsetSelection.setLearningSet(lstFSETrainConfigurationListing)
tmpSubsetSelection.setValidationSet(
lstFSETrainConfigurationListing
) # Following FSE paper, Learning set is resued as 'validation set'.
tmpSubsetSelection.learn()
showInfluenceModel = False
if showInfluenceModel:
printInfluenceModel(tmpSubsetSelection)
lstFSETestConfigurationListing = transformToFSEFormat(
testConfigurationsList, vmX264)
setNFPValuesForConfigurationList(lstFSETestConfigurationListing,
testConfigurationsList,
traceExecutionTimesSummaries)
averagingTestSet = False
if averagingTestSet == True:
lstMeasuredValues = [
x.getNfpValue() for x in lstFSETestConfigurationListing
]
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
showVerboseDebugging = False
if showVerboseDebugging:
for indexOffset in range(5):
print("Configuration {0}".format(
testConfigurationsList[indexOffset]))
print("Configuration wrt Options {0}".format([
x.name
for x in lstFSETestConfigurationListing[indexOffset].
dctBinaryOptionValues.keys()
]))
print("Measured Value {0}".format(
lstMeasuredValues[indexOffset]))
print("Estimated Value {0}".format(
lstEstimatedValues[indexOffset]))
else:
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
lstMeasuredValues = []
for aConfId in testConfigurationsList:
for repetitionId in range(1, 11):
executedTimes = traceExecutionTimesSummaries[(aConfId,
repetitionId)]
lstMeasuredValues.append(executedTimes)
tmpLstEstimatedValues = []
for estimatedVal in lstEstimatedValues:
for i in range(0, 10):
tmpLstEstimatedValues.append(estimatedVal)
lstEstimatedValues = tmpLstEstimatedValues
lstMeasuredValuesNp = np.array(lstMeasuredValues)
lstEstimatedValuesNp = np.array(lstEstimatedValues)
MAETestMean, MAETestStd = mean_absolute_error_and_stdev_eff(
lstMeasuredValuesNp, lstEstimatedValuesNp)
MEANTestVal = np.mean(lstMeasuredValuesNp)
NormalizedMae = 100 * (MAETestMean / MEANTestVal)
print("MAE_TEST_MEAN, MAE_TEST_STD, MEAN_TEST, NOMRALIZED_MAE (%) ")
print("{0},\t {1},\t {2},\t {3}".format(MAETestMean, MAETestStd,
MEANTestVal, NormalizedMae))