def extractTracesFromSinglePKL(TrainingSetConfigurations, TrainingConfSize,
TrainingConfFilename, OutputFilenamem,
verboseDebug):
"""
Receive Encoding of traces for each configuration
"""
allTracesAutonomoose = loadObjectFromPickle(
getSingleFilenameWithAllTraces())
GlobalTmpArray = []
for configurationId in TrainingSetConfigurations:
print("Processing {0}".format(configurationId))
# print ("Configuration {0}, Number of Traces {1} ".format(configurationId, len(allTracesAutonomoose[configurationId])))
lstAllTracesForConf = allTracesAutonomoose[configurationId]
dctTransitionsToExecutions = {}
for anAutonomoseExecutionTrace in lstAllTracesForConf:
#Each trace is AutonomooseTraces.ExecutionTraceAutonomoose
anAutonomoseExecutionTrace.addExecutedTransitionsToDictionary(
dctTransitionsToExecutions)
# print ("After processing an execution of type {0}, we have a dicitonary with {1} elements".format(type(anAutonomoseExecutionTrace), \
# sum([len(dctTransitionsToExecutions[x]) for x in dctTransitionsToExecutions.keys()])))
print(dctTransitionsToExecutions.keys())
GlobalTmpArray.append(dctTransitionsToExecutions)
saveObjectToPickleFile(OutputFilename, GlobalTmpArray)
def summarizeTracesX264SpecialFromResults(
pklOuptputFilename,
baseFilenameTemplate="news/results/rq2_predictions_rep_{0}.csv"):
"""
Summarize Traces X264 from both results and from reading through traces (for the configurations in the training set)
Get Input data for files News and Container.
"""
dctAllTracesExecutionTimes = {}
showNextInserts = False
for testTrainSplit in range(1, 11):
fdFilename = baseFilenameTemplate.format(testTrainSplit)
print("Starting openning {0}".format(fdFilename))
fdAll = open(fdFilename, "r")
lineIndex = 0
seenConfigurations = 0
for line in fdAll:
if lineIndex == 0:
lineIndex = lineIndex + 1
continue
splitInParts = line.strip().split(",")
if lineIndex == 18440 and testTrainSplit == 1:
print(splitInParts)
confId, timeTaken = splitInParts[0], splitInParts[1]
if (int(confId), (seenConfigurations +
1)) not in dctAllTracesExecutionTimes.keys():
seenConfigurations = seenConfigurations + 1
if showNextInserts:
print("Will insert {0}--{1}".format(
confId, seenConfigurations))
dctAllTracesExecutionTimes[(
int(confId), seenConfigurations)] = float(timeTaken)
if seenConfigurations == 10:
seenConfigurations = 0
lineIndex = lineIndex + 1
if lineIndex == 18441:
break
print("After full search we have {0} keys".format(
len(dctAllTracesExecutionTimes.keys())))
if len(dctAllTracesExecutionTimes.keys()) >= 23030:
showNextInserts = True
for a, b in dctAllTracesExecutionTimes.keys():
if (a not in range(0, 2304)) and (b not in range(1, 11)):
print("a={0} b={1}".format(a, b))
pickleFacade.saveObjectToPickleFile(pklOuptputFilename,
dctAllTracesExecutionTimes)
def summarizeTracesX264(pklOuptputFilename,
pklOuptputPerTransitionFilename,
onlyPerTransition=False):
"""
Read all traces from the source folder (already set) and summarize their overall execution time, # of counts per transition, and total time per transition.
Creates two PKL Files. First one is dictionary mapping confId, repetitionId to overal execution time.
Second one is Chained Dictionary mapping Conf -- > rep --> transitionId --> (Count, TimeTakenTotal)
Parameters
----------
pklOutput : string
Filename where to store results.
onlyPerTransition: special flag to indicate not to save per trace totals
"""
allConfigurationsIds = range(0, 2304)
dctAllTracesExecutionTimes = {}
dctConfTodctRepTodctTransitionToTimes = {}
for aConfId in allConfigurationsIds:
print("Summarizing Configuration {0}".format(aConfId))
dctConfTodctRepTodctTransitionToTimes[aConfId] = {}
for repetitionId in range(1, 11):
dctTransitionToTimeTamekn = sumTimeTakenPerTransitionFromConfigurationAndRep(
aConfId, repetitionId)
if (not onlyPerTransition):
timeTakenByTraceAddition = sum([
dctTransitionToTimeTamekn[x][MLConstants.tupleTimeOffset]
for x in dctTransitionToTimeTamekn.keys()
])
dctAllTracesExecutionTimes[(
aConfId, repetitionId)] = timeTakenByTraceAddition
dctConfTodctRepTodctTransitionToTimes[aConfId][
repetitionId] = dctTransitionToTimeTamekn
# print(timeTakenByTraceAddition)
# print (dctAllTracesExecutionTimes.keys())
# print (dctConfTodctRepTodctTransitionToTimes.keys())
if (onlyPerTransition):
pickleFacade.saveObjectToPickleFile(
pklOuptputPerTransitionFilename,
dctConfTodctRepTodctTransitionToTimes)
else:
pickleFacade.saveObjectToPickleFile(pklOuptputFilename,
dctAllTracesExecutionTimes)
pickleFacade.saveObjectToPickleFile(
pklOuptputPerTransitionFilename,
dctConfTodctRepTodctTransitionToTimes)
loadObjectFromPickle(confTrainFilename)
if SubjectSystem == MLConstants.x264Name:
listTransitionToSample = getAllTransitionsIdsList()
elif SubjectSystem == MLConstants.autonomooseName:
listTransitionToSample = getListOfAvailableTransitionsAutonomoose(trainDataset)
RegressorList = []
for transitionId in listTransitionToSample:
if transitionId in bestRegressorPerTransition.keys():
newRegressor = crateRegressorWrapperFromTuple(transitionId, bestRegressorPerTransition[transitionId])
else:
continue
if SubjectSystem == MLConstants.x264Name:
YSet = extractLinearArrayTimeTakenForSingleTransition(trainDataset, newRegressor.getTransitionId())
else:
YSet = getSetOfExecutionTimesAutonomoose(trainDataset, newRegressor.getTransitionId(), trainOrderedConfs)
FitTrainDataFroRegressor(newRegressor, trainOrderedConfs, YSet, SubjectSystem)
RegressorList.append(newRegressor)
saveObjectToPickleFile(RegressorOutputFilename, RegressorList)
def extractAndSampleBySectionsFromTraces(TrainingSetConfigurations,
TrainingConfSize,
TrainingConfFilename, OutputFilenamem,
verboseDebug):
"""
Generate a file that includes up to 500 000 executions of each transition for each configuraiton in TrainingSetConfigurations.
"""
if verboseDebug:
# free only exists in linux
print("Memory Information at Program Launch ")
call(["free", "-h"])
globalCounts = {}
for outerPart in range(0, NUM_TEMPORARY_PARTS):
transitionArrayOfDictionary = []
smallSet = TrainingSetConfigurations[outerPart * 20:(outerPart * 20) +
20]
if verboseDebug:
print("At subloop {0}, memory status :".format(outerPart))
call(["free", "-h"])
for configurationId in smallSet:
for repId in range(1, NUM_REPETITIONS_PER_TRACE + 1):
traceFilename = getFilenameFromConfigurationAndRepetition(
configurationId, repId)
AllTransitions = extractTransitionToBagOfTimesDictionaryFromTraceFile(
traceFilename)
transitionArrayOfDictionary.append(AllTransitions)
if verboseDebug:
print(
"Possible Peak subloop memory at loop {0}".format(outerPart))
call(["free", "-h"])
mergedDictionary = conjoinRepetedDictionaries(
transitionArrayOfDictionary)
transitionArrayOfDictionary = []
if verboseDebug:
print(
"Possible Peak subloop memory at loop {0}".format(outerPart))
call(["free", "-h"])
allCounts = calculatePerTransitionsCounts(mergedDictionary)
if verboseDebug:
print(allCounts)
addCountDictionaries(globalCounts, allCounts)
if verboseDebug:
print("Will save Dict of length = {0}".format(
len(mergedDictionary)))
saveObjectToPickleFile(temporaryFilenameTemplate.format(outerPart),
mergedDictionary)
transitionArrayOfDictionary = []
AllTransitions = []
mergedDictionary = []
allCounts = {}
if verboseDebug:
print("Memory After Clean Up")
call(["free", "-h"])
if verboseDebug:
print("Completed")
print("globalCounts: {0}".format(globalCounts))
if verboseDebug:
print("Final Memory Before Shutdown")
call(["free", "-h"])
GlobalTmpFinalArray = []
for outerIndex in range(0, NUM_TEMPORARY_PARTS):
if verboseDebug:
print("Memory Start Loading Loop at {0}".format(outerIndex))
call(["free", "-h"])
unsampledMergedDictArray = loadObjectFromPickle(
temporaryFilenameTemplate.format(outerIndex))
if verboseDebug:
print("Loading Memory Peak I at {0}".format(outerIndex))
call(["free", "-h"])
TmpFinalArrayDict = downSampleToNewMaxExecutions(
unsampledMergedDictArray, actualCountsDictionary=globalCounts)
if verboseDebug:
print("Loading Memory Peak II at {0}".format(outerIndex))
call(["free", "-h"])
unsampledMergedDictArray = []
GlobalTmpFinalArray.extend(TmpFinalArrayDict)
TmpFinalArrayDict = []
if verboseDebug:
print("Reloading Memory Final at {0}".format(outerIndex))
call(["free", "-h"])
if verboseDebug:
print("Saving Final Dictionary of length {0}".format(
len(GlobalTmpFinalArray)))
saveObjectToPickleFile(OutputFilename, GlobalTmpFinalArray)
setBaseTracesSourceFolder(TraceSourceFolder)
if SubjectSystem == MLConstants.x264Name:
TestsetConfigurationSize = 2304 - TrainingConfSize
TrainingSetConfigurations = train_test_split(getAllPossibleIds(MLConstants.x264Id), getAllPossibleIds(MLConstants.x264Id),\
train_size=TrainingConfSize, test_size=TestsetConfigurationSize)[0]
elif SubjectSystem == MLConstants.autonomooseName:
TestsetConfigurationSize = 32 - TrainingConfSize
TrainingSetConfigurations = train_test_split(getAllPossibleIds(MLConstants.autonomooseId), getAllPossibleIds(MLConstants.autonomooseId),\
train_size=TrainingConfSize, test_size=TestsetConfigurationSize)[0]
saveObjectToPickleFile(TrainingConfFilename, TrainingSetConfigurations)
if SubjectSystem == MLConstants.x264Name:
extractAndSampleBySectionsFromTraces(TrainingSetConfigurations,
TrainingConfSize,
TrainingConfFilename,
OutputFilename, verboseDebug)
elif SubjectSystem == MLConstants.autonomooseName:
extractTracesFromSinglePKL(TrainingSetConfigurations, TrainingConfSize,
TrainingConfFilename, OutputFilename,
verboseDebug)
ConfsAleadySampledExtra = loadObjectFromPickle(
preExistingSampledConfsFilenameExtra)
DatasetAlreadySampledExtra = loadObjectFromPickle(
preExistingSampledDatasetFilenameExtra)
else:
allTracesAutonomoose = loadObjectFromPickle(
getSingleFilenameWithAllTraces())
confsTest = getComplementSet(
confsTrain, totalEleements=MLConstants.AutonomooseConfCount)
saveObjectToPickleFile(assesmentConfsFilename, confsTest)
DictionaryArray = []
counter = 0
for confId in confsTest:
if SubjectSystem == MLConstants.x264Name:
if counter % 10 == 0:
print("Sampling {0} out of {1}".format(
counter, len(confsTest)))
if (useDifferentialSampling and confId in ConfsAleadySampled):
indexOfConfid = ConfsAleadySampled.index(confId)
DictionaryArray.append(DatasetAlreadySampled[indexOfConfid])
OutputFilename = sys.argv[2]
else:
print(
"Requires two arguments: source folder with traces, and filename to store pkl of traces."
)
exit()
dictConfToTraces = {}
for confId in range(0, 32):
print("Conf {0}".format(confId))
for rep in range(0, 10):
rosbagFilename = "{0}track_transition_counts-configuration-{1}-repetition-{2}.bag".format(
InputFolder, confId, rep)
executionTraceBag = rosbag.Bag(rosbagFilename, 'r')
newTrace = AutonomooseTraces.ExecutionTraceAutonomoose(
executionTraceBag, confId)
if confId in dictConfToTraces.keys():
dictConfToTraces[confId].append(newTrace)
else:
dictConfToTraces[confId] = [newTrace]
print(dictConfToTraces.keys())
saveObjectToPickleFile(OutputFilename, dictConfToTraces)