def extractTracesFromSinglePKL(TrainingSetConfigurations, TrainingConfSize,
TrainingConfFilename, OutputFilenamem,
verboseDebug):
"""
Receive Encoding of traces for each configuration
"""
allTracesAutonomoose = loadObjectFromPickle(
getSingleFilenameWithAllTraces())
GlobalTmpArray = []
for configurationId in TrainingSetConfigurations:
print("Processing {0}".format(configurationId))
# print ("Configuration {0}, Number of Traces {1} ".format(configurationId, len(allTracesAutonomoose[configurationId])))
lstAllTracesForConf = allTracesAutonomoose[configurationId]
dctTransitionsToExecutions = {}
for anAutonomoseExecutionTrace in lstAllTracesForConf:
#Each trace is AutonomooseTraces.ExecutionTraceAutonomoose
anAutonomoseExecutionTrace.addExecutedTransitionsToDictionary(
dctTransitionsToExecutions)
# print ("After processing an execution of type {0}, we have a dicitonary with {1} elements".format(type(anAutonomoseExecutionTrace), \
# sum([len(dctTransitionsToExecutions[x]) for x in dctTransitionsToExecutions.keys()])))
print(dctTransitionsToExecutions.keys())
GlobalTmpArray.append(dctTransitionsToExecutions)
saveObjectToPickleFile(OutputFilename, GlobalTmpArray)
def generatePerfumeTracesAutonomoose(LstConfigurationIds, sourceFolder):
"""
Extract a transe for a single Autonomoose configuration (or list of Autonomoose configurations.)
TODO -- how to partition an Autonomoose trace into "mini" traces.
"""
if sourceFolder.find("First") != -1:
tmpTraceShorthand = "first"
elif sourceFolder.find("Second") != -1:
tmpTraceShorthand = "second"
else:
tmpTraceShorthand = "third"
allTraces = loadObjectFromPickle(getSingleFilenameWithAllTraces())
for configurationId in LstConfigurationIds:
AutonomooseTraceForConfiguration = allTraces[configurationId][0]
tmpTransList = AutonomooseTraceForConfiguration.getTransitionList()
filenameToOutputTo = "PerfumeControl/Traces/autonomoose_{0}_configuration_{1}".format(
tmpTraceShorthand, configurationId)
countAndPrintAutonomooseLoops(configurationId, tmpTransList,
filenameToOutputTo)
def chooseConfigurationsPresentInAllTestSets(traceDirectories=[]):
chosenConfs = set([])
trainingSetsAcrossAsSets = []
trainingSetsAcrossAsLists = []
for traceFolder in traceDirectories:
q = loadObjectFromPickle(traceFolder + "test_conf_rep_1.pkl")
trainingSetsAcrossAsSets.append(set(q))
trainingSetsAcrossAsLists.append(q)
while (len(chosenConfs) < 10):
potentialConfiguration = random.choice(trainingSetsAcrossAsLists[0])
if (potentialConfiguration not in chosenConfs) and \
(potentialConfiguration in trainingSetsAcrossAsSets[1]) and \
(potentialConfiguration in trainingSetsAcrossAsSets[2]):
chosenConfs.add(potentialConfiguration)
return chosenConfs
print (len(inputParameters))
print(" Incorrect Usage. Requires five parameters: Subject System, sampleTrainFilename, confTrainFilename, CvResultsFilenamem, RegressorOutputFilename")
exit()
return SubjectSystem, sampleTrainFilename, confTrainFilename, CVResultsFilename, RegressorOutputFilename
if __name__ == "__main__":
SubjectSystem, sampleTrainFilename, confTrainFilename, CVResultsFilename, RegressorOutputFilename = parseRuntimeParemeters(sys.argv)
bestRegressorPerTransition = getBestMethodPerTransition(CVResultsFilename)
trainDataset, trainOrderedConfs = loadObjectFromPickle(sampleTrainFilename), \
loadObjectFromPickle(confTrainFilename)
if SubjectSystem == MLConstants.x264Name:
listTransitionToSample = getAllTransitionsIdsList()
elif SubjectSystem == MLConstants.autonomooseName:
listTransitionToSample = getListOfAvailableTransitionsAutonomoose(trainDataset)
RegressorList = []
for transitionId in listTransitionToSample:
if transitionId in bestRegressorPerTransition.keys():
newRegressor = crateRegressorWrapperFromTuple(transitionId, bestRegressorPerTransition[transitionId])
else:
else:
print("Incorrect usage - requires 6 filenames parameters: Subject System, train data set," \
"train configurations, results of CV, test dataset, test configurations")
exit(0)
return SubjectSystem, trainDatasetFilename, trainConfFilename, CvResultsFilename, testDatasetFilename, testConfFilename
if __name__ == "__main__":
SubjectSystem, trainDatasetFilename, trainConfFilename, CvResultsFilename, testDatasetFilename, testConfFilename = parseRuntimeParemeters(
sys.argv)
bestRegressorPerTransition = getBestMethodPerTransition(CvResultsFilename)
testDataset, testOrderedConfs, trainDataset, trainOrderedConfs = loadObjectFromPickle(testDatasetFilename), \
loadObjectFromPickle(testConfFilename), loadObjectFromPickle(trainDatasetFilename), loadObjectFromPickle(trainConfFilename)
if SubjectSystem == MLConstants.x264Name:
transitionIdList = getAllTransitionsIdsList()
else:
# Autonomoose
transitionIdList = getListOfAvailableTransitionsAutonomoose(
trainDataset)
print(
"Transition Id, YMean_Train, Y_Std_Train, MAE_Train, MAE_Std_Train, YMean_Test, Y_Std_Test, MAE_Test, MAE_Std_Test"
)
for transitionId in transitionIdList:
def analyzeOverallExecutionTimesAutonomoose(regressorsArray,
testConfigurationsList,
transitionToRegressorMapping,
transitionToConfArrayTimeTaken):
"""
Autonomoose all traces are in a single file.
Calculate actual execution time for trace of autonomoose, as well as predicted execution time.
"""
listActualTimes = []
listPredictedTimes = []
allTraces = loadObjectFromPickle(getSingleFilenameWithAllTraces())
# print (len(testConfigurationsList))
for aConfId, offsetIndex in zip(testConfigurationsList,
range(0, len(testConfigurationsList))):
#print("Computing for Configuration {0} with offset Index = {1}".format(aConfId, offsetIndex))
for repetitionId in range(0, 10):
CurrentExecutionTrace = allTraces[aConfId][repetitionId]
actualExecutionTime = getOverallRealTimeForASingleTraceAutonomoose(
CurrentExecutionTrace, aConfId)
transitionsCounts = CurrentExecutionTrace.getPerTransitionCounts()
#print(transitionsCounts)
predictedTimeTaken = 0.0
for aTransitionId in transitionsCounts.keys():
predictedTimeTaken = predictedTimeTaken + transitionsCounts[
aTransitionId] * transitionToConfArrayTimeTaken[
aTransitionId][offsetIndex]
if not (type(predictedTimeTaken) == float):
print("{0},{1},{2}".format(aConfId, actualExecutionTime,
predictedTimeTaken[0]))
else:
print("{0},{1},{2}".format(aConfId, actualExecutionTime,
predictedTimeTaken))
listActualTimes.append(actualExecutionTime)
listPredictedTimes.append(predictedTimeTaken)
npActualTimes = np.array([np.array([x]) for x in listActualTimes])
npPredictedTimes = np.array(listPredictedTimes)
meanMAE, stdMAE = mean_absolute_error_and_stdev_eff(
npActualTimes, npPredictedTimes)
print("Mean_MAE,MAE_STDEV")
if meanMAE.shape == (1, ):
print("{0},{1}".format(meanMAE[0], stdMAE[0]))
else:
print("{0},{1}".format(meanMAE, stdMAE))
print("Mean_MAE,MAE_STDEV")
if meanMAE.shape == (1, ):
print("{0},{1}".format(meanMAE[0], stdMAE[0]))
else:
print("{0},{1}".format(meanMAE, stdMAE))
if __name__ == "__main__":
SubjectSystem, TraceSourceFolder, regressorInputFilename, testConfFilename = parseRuntimeParemeters(
sys.argv)
setBaseTracesSourceFolder(TraceSourceFolder)
regressorsArray, testConfigurationsList = loadObjectFromPickle(
regressorInputFilename), loadObjectFromPickle(testConfFilename)
transitionToRegressorMapping = getRegressorToTransitionIdMapping(
regressorsArray)
transitionToConfArrayTimeTaken = {}
for transitionId in transitionToRegressorMapping.keys():
if SubjectSystem == MLConstants.x264Name:
transitionToConfArrayTimeTaken[transitionId] = getPredictionsForTransitionsOnConfigurationList(testConfigurationsList, \
regressorsArray, transitionToRegressorMapping, transitionId)
else:
transitionToConfArrayTimeTaken[transitionId] = getPredictionsForTransitionsOnConfigurationList(testConfigurationsList, \
regressorsArray, transitionToRegressorMapping, transitionId, generateBitsetForOneConfigurationAutonomoose)
print("Configuration_Id, Actual Execution Time, Predicted Execution Time")
for transitionId in listTransitionsToSample:
YSet = getSetOfExecutionTimesAutonomoose(transitionData, transitionId,
trainingSetConfigurations)
learnAndCrossValidateForATransitionGeneric(
transitionId, trainingSetConfigurations, kf, YSet,
generateBitsetForOneConfigurationAutonomoose)
if __name__ == "__main__":
SubjectSystem, confFilename, inputFilename = parseRuntimeParemeters(
sys.argv)
trainingSetConfigurations = loadObjectFromPickle(confFilename)
transitionArrayOfDictionary = loadObjectFromPickle(inputFilename)
assert (len(trainingSetConfigurations) == len(transitionArrayOfDictionary))
printOutputHeader()
if SubjectSystem == MLConstants.x264Name:
learnFromTraininingSetX264(trainingSetConfigurations,
transitionArrayOfDictionary)
elif SubjectSystem == MLConstants.autonomooseName:
learnFromTraininingSetAutonomoose(trainingSetConfigurations,
def extractAndSampleBySectionsFromTraces(TrainingSetConfigurations,
TrainingConfSize,
TrainingConfFilename, OutputFilenamem,
verboseDebug):
"""
Generate a file that includes up to 500 000 executions of each transition for each configuraiton in TrainingSetConfigurations.
"""
if verboseDebug:
# free only exists in linux
print("Memory Information at Program Launch ")
call(["free", "-h"])
globalCounts = {}
for outerPart in range(0, NUM_TEMPORARY_PARTS):
transitionArrayOfDictionary = []
smallSet = TrainingSetConfigurations[outerPart * 20:(outerPart * 20) +
20]
if verboseDebug:
print("At subloop {0}, memory status :".format(outerPart))
call(["free", "-h"])
for configurationId in smallSet:
for repId in range(1, NUM_REPETITIONS_PER_TRACE + 1):
traceFilename = getFilenameFromConfigurationAndRepetition(
configurationId, repId)
AllTransitions = extractTransitionToBagOfTimesDictionaryFromTraceFile(
traceFilename)
transitionArrayOfDictionary.append(AllTransitions)
if verboseDebug:
print(
"Possible Peak subloop memory at loop {0}".format(outerPart))
call(["free", "-h"])
mergedDictionary = conjoinRepetedDictionaries(
transitionArrayOfDictionary)
transitionArrayOfDictionary = []
if verboseDebug:
print(
"Possible Peak subloop memory at loop {0}".format(outerPart))
call(["free", "-h"])
allCounts = calculatePerTransitionsCounts(mergedDictionary)
if verboseDebug:
print(allCounts)
addCountDictionaries(globalCounts, allCounts)
if verboseDebug:
print("Will save Dict of length = {0}".format(
len(mergedDictionary)))
saveObjectToPickleFile(temporaryFilenameTemplate.format(outerPart),
mergedDictionary)
transitionArrayOfDictionary = []
AllTransitions = []
mergedDictionary = []
allCounts = {}
if verboseDebug:
print("Memory After Clean Up")
call(["free", "-h"])
if verboseDebug:
print("Completed")
print("globalCounts: {0}".format(globalCounts))
if verboseDebug:
print("Final Memory Before Shutdown")
call(["free", "-h"])
GlobalTmpFinalArray = []
for outerIndex in range(0, NUM_TEMPORARY_PARTS):
if verboseDebug:
print("Memory Start Loading Loop at {0}".format(outerIndex))
call(["free", "-h"])
unsampledMergedDictArray = loadObjectFromPickle(
temporaryFilenameTemplate.format(outerIndex))
if verboseDebug:
print("Loading Memory Peak I at {0}".format(outerIndex))
call(["free", "-h"])
TmpFinalArrayDict = downSampleToNewMaxExecutions(
unsampledMergedDictArray, actualCountsDictionary=globalCounts)
if verboseDebug:
print("Loading Memory Peak II at {0}".format(outerIndex))
call(["free", "-h"])
unsampledMergedDictArray = []
GlobalTmpFinalArray.extend(TmpFinalArrayDict)
TmpFinalArrayDict = []
if verboseDebug:
print("Reloading Memory Final at {0}".format(outerIndex))
call(["free", "-h"])
if verboseDebug:
print("Saving Final Dictionary of length {0}".format(
len(GlobalTmpFinalArray)))
saveObjectToPickleFile(OutputFilename, GlobalTmpFinalArray)
"Incorrect usage - requires >= 5 filenames parameters: Subject System, Trace Folder, train conf pkl, test conf pkl, and test conf sampled, . Optional 2 (or 4) more for differential sampling, preceeded by subjectName and optional baseName for moose "
)
exit(0)
return SubjectSystem, TraceSourceFolder, confTrainFilename, assesmentConfsFilename, assesmentSampledFilename, \
useDifferentialSampling, preExistingSampledConfsFilename, preExistingSampledDatasetFilename, useDifferentialSamplingExtra, preExistingSampledConfsFilenameExtra, \
preExistingSampledDatasetFilenameExtra
if __name__ == "__main__":
SubjectSystem, TraceSourceFolder, confTrainFilename, assesmentConfsFilename, assesmentSampledFilename, \
useDifferentialSampling, preExistingSampledConfsFilename, preExistingSampledDatasetFilename, useDifferentialSamplingExtra, preExistingSampledConfsFilenameExtra, \
preExistingSampledDatasetFilenameExtra = parseRuntimeParemeters(sys.argv)
confsTrain = loadObjectFromPickle(confTrainFilename)
setBaseTracesSourceFolder(TraceSourceFolder)
if SubjectSystem == MLConstants.x264Name:
confsTest = getComplementSet(confsTrain)
# X264 only preprocessing.
dictRatios = getTestSetSamplingRatiosDict()
if useDifferentialSampling:
ConfsAleadySampled = loadObjectFromPickle(
preExistingSampledConfsFilename)
def analyzeAutonomooseFSE(trainConfigurationList, testConfigurationsList):
"""
Analyze execution time of autonomooose
Parameters
-----------
traceExecutionTimesSummaries : Dictionary
Maps tuples (confId, repetitionId) to execution time (microseconds)
trainConfigurationList : List of integers
testConfigurationsList : List of integers
Notes
------
1. Open all tracess files.
BooleanOptions = [("BEHAVIOR", 4 ), \
("OCCUPANCY", 2),
("WAYPOINTS", 1)]
Behavior Planner
Occupancy or Mockupancy Planner
Waypoints Collection
Dyn. Object Tracking
Dyn. Car Tracking.
Dyn. Person Tracking
TODO - REFACTOR to join it into analyzeX264FSE -- strategy pattern.
"""
allTraces = loadObjectFromPickle(getSingleFilenameWithAllTraces())
vmAutonomoose = generateFullAutonomooseVariabilityModel()
InfModelAutonomoose = InfluenceModels.InfluenceModel(vmAutonomoose)
TmpMLSettings = MLSettings.MLSettings()
TmpMLSettings.useBackward = True
lstFSETrainConfigurationListing = transformToFSEFormat(trainConfigurationList, vmAutonomoose, \
ConfigurationIdToBitsetTransformer=generateBitsetForOneConfigurationAutonomoose, \
BitsetToFseTransformer=transformSingleConfigurationToAutonomooseFSE)
setNFPValuesForConfigurationListAutonomoose(
lstFSETrainConfigurationListing, trainConfigurationList, allTraces)
tmpSubsetSelection = FeatureSubsetSelection.FeatureSubsetSelection(
InfModelAutonomoose, TmpMLSettings)
tmpSubsetSelection.setLearningSet(lstFSETrainConfigurationListing)
tmpSubsetSelection.setValidationSet(
lstFSETrainConfigurationListing
) # Following FSE paper, Learning set is resued as -- 'validation set' --
tmpSubsetSelection.learn()
showInfluenceModel = False
if showInfluenceModel:
printInfluenceModel(tmpSubsetSelection)
lstFSETestConfigurationListing = transformToFSEFormat(testConfigurationsList, vmAutonomoose, \
ConfigurationIdToBitsetTransformer=generateBitsetForOneConfigurationAutonomoose, \
BitsetToFseTransformer=transformSingleConfigurationToAutonomooseFSE)
#
#
#
# CALCULATING NMAE WITHOUT USING AVERAGING.
averagingTestSet = False
if averagingTestSet == True:
setNFPValuesForConfigurationListAutonomoose(
lstFSETestConfigurationListing, testConfigurationsList, allTraces)
lstMeasuredValues = [
x.getNfpValue() for x in lstFSETestConfigurationListing
]
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
else:
lstMeasuredValues = []
for aConfId in testConfigurationsList:
for repetitionId in range(0, 10):
executedTimes = getOverallRealTimeForASingleTraceAutonomoose(
allTraces[aConfId][repetitionId], aConfId)
lstMeasuredValues.append(executedTimes)
lstEstimatedValues = [
tmpSubsetSelection.infModel.estimate(x)
for x in lstFSETestConfigurationListing
]
tmpLstEstimatedValues = []
for estimatedVal in lstEstimatedValues:
for i in range(0, 10):
tmpLstEstimatedValues.append(estimatedVal)
lstEstimatedValues = tmpLstEstimatedValues
lstMeasuredValuesNp = np.array(lstMeasuredValues)
lstEstimatedValuesNp = np.array(lstEstimatedValues)
MAETestMean, MAETestStd = mean_absolute_error_and_stdev_eff(
lstMeasuredValuesNp, lstEstimatedValuesNp)
MEANTestVal = np.mean(lstMeasuredValuesNp)
NormalizedMae = 100 * (MAETestMean / MEANTestVal)
print("MAE_TEST_MEAN, MAE_TEST_STD, MEAN_TEST, NOMRALIZED_MAE (%) ")
print("{0},\t {1},\t {2},\t {3}".format(MAETestMean, MAETestStd,
MEANTestVal, NormalizedMae))
if __name__ == "__main__":
"""
Execute FSE paper for X264 / Autonomooose.
Testing --
akiyo/traceExecutionTimesForAll.pkl
"""
SubjectSystem, trainConfFilename, testConfFilename, traceSummarizedTimesFilenameOrDirectoryName = parseRuntimeParemeters(
sys.argv)
trainConfigurationList, testConfigurationsList = loadObjectFromPickle(trainConfFilename), \
loadObjectFromPickle(testConfFilename)
if SubjectSystem == MLConstants.x264Name:
traceExecutionTimesSummaries = loadObjectFromPickle(
traceSummarizedTimesFilenameOrDirectoryName)
analyzeX264FSE(trainConfigurationList, testConfigurationsList,
traceExecutionTimesSummaries)
else:
#traceSummarizedTimesFilenameOrDirectoryName "../FullTraces/autonomooseFirst/"
setBaseTracesSourceFolder(traceSummarizedTimesFilenameOrDirectoryName)
