def merge_testdata(self, _dirpath, _args):
_parts = _args.numParts
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=r'\d+',
_sort=True)
progress = tqdm(desc='Collecting data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
output = open(os.path.join(item['path'], "testdata.csv"), "w")
for partID in range(1, _parts + 1):
datafile = os.path.join(item['path'],
'testdata_part%02d.csv' % partID)
title, lines = self.load_data(datafile)
if title is None:
print("Not found the file: " + datafile)
# data lines contain '\n'
if partID == 1:
output.write(title)
for line in lines:
output.write(line)
output.close()
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
def merge_rt_data(self, _dirpath, _args):
_nParts = _args.numParts
_targetName = _args.targetName if _args.targetName is not None else '_roundtrip'
_startNum = _args.startNum if _args.startNum is not None else 1
_outputName = _args.outputName if _args.outputName is not None else 'result.csv'
_selected = _args.selectedRuns
selectedRuns = None
if _selected is not None:
selectedRuns = _selected.split(',')
selectedRuns = [int(var) for var in selectedRuns]
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=r'\d+',
_sort=True)
progress = tqdm(desc='Collecting roundtrip data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
if selectedRuns is not None and int(
item['Run']) not in selectedRuns:
progress.update(1)
progress.set_postfix_str(item['path'])
continue
numLines = 0
outputname = os.path.join(item['path'],
"%s/%s" % (_targetName, _outputName))
output = open(outputname, "w")
for partID in range(_startNum, _nParts + 1):
datafile = os.path.join(
item['path'],
'%s/result_part%02d.csv' % (_targetName, partID))
title, lines = self.load_data(datafile)
if title is None:
print("Not found the file: " + datafile)
# data lines contain '\n'
if partID == _startNum:
output.write(title)
for line in lines:
output.write(line)
numLines += len(lines)
output.close()
if numLines < 40000:
print("Not enough number of results: " + outputname)
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
def merge_p2_model(self, _dirpath, _outputname, _args):
# parameter passing
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_exptions = _args.exceptions
_targetName = _args.targetName if _args.targetName is not None else '_phase2'
_nUpdates = _args.nUpdates if _args.nUpdates is not None else 100
_timeQuanta = _args.timeQuanta if _args.timeQuanta is not None else 0.01
_special = _args.special if _args.special is not None else 0
# listing target directories
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
taskInfo = TaskDescriptor.load_fromFile(
os.path.join(targets[0]['path'], 'input.csv'), _timeQuanta)
uncertainTasks = TaskDescriptor.getUncertainTasks(taskInfo)
# output
output = open(_outputname, "w")
output.write("Run,ExecutionTime(s),nTerms,Probability,BestSize,%s\n" %
(','.join(["T%d" % t for t in uncertainTasks])))
# progressing
progress = tqdm(desc='Collecting data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
if _exptions is not None and item['Run'] in _exptions:
progress.update(1)
progress.set_postfix_str(item['path'])
continue
try:
self.process_onerun(item, _nUpdates, _targetName,
uncertainTasks, taskInfo, _timeQuanta,
output, _special)
except Exception as e:
print('Filed to get information: run%02d' % int(item['Run']))
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
output.close()
def check_roundtrip(self, _dirpath, _args):
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_targetName = _args.targetName if _args.targetName is not None else r'_roundtrip/result.csv'
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
for item in targets:
ret = self.load_rtdata(os.path.join(item['path'], _targetName))
if ret is None or ret['title'] is False:
print('%s Run %02d: Not found the roundtrip result' %
(_dirpath, int(item['Run'])))
else:
print("%s Run %02d: %d result (ratio: %.4f)" %
(_dirpath, int(item['Run']), ret['count'], ret['ratio']))
def check_testdata(self, _dirpath, _args):
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_targetName = _args.targetName if _args.targetName is not None else r'testdata.csv'
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
for item in targets:
ret = self.load_testdata(os.path.join(item['path'], _targetName))
if ret is None:
print('%s Run %02d: Not found the test data' %
(_dirpath, int(item['Run'])))
else:
sumNum = ret['positive'] + ret['negative']
print("%s Run %02d: %d testdata (positive: %d, negative: %d)" %
(_dirpath, int(item['Run']), sumNum, ret['positive'],
ret['negative']))
def merge_execinfo(self, _dirpath, _outputname, _args):
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_targetName = _args.targetName if _args.targetName is not None else 'result.txt'
_exptions = _args.exceptions
output = open(_outputname, "w")
output.write(
"Run,Total(s),InitHeap(MB),UsedHeap(MB),CommitHeap(MB),MaxHeap(MB),MaxNonHeap(MB)\n"
)
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
progress = tqdm(desc='Collecting data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
if _exptions is not None and item['Run'] in _exptions:
progress.update(1)
progress.set_postfix_str(item['path'])
continue
timeInfo = self.load_time(os.path.join(item['path'], _targetName))
if timeInfo is None:
output.write("%d,NA, NA,NA,NA,NA,NA\n" % int(item['Run']))
else:
output.write("%d,%f, %f,%f,%f,%f,%f\n" %
(int(item['Run']), timeInfo['TotalExecutionTime'],
timeInfo['InitHeap'], timeInfo['usedHeap'],
timeInfo['commitHeap'], timeInfo['MaxHeap'],
timeInfo['MaxNonHeap']))
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
output.close()
def merge_p2_test(self, _dirpath, _outputname, _args):
# parameter passing
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_exptions = _args.exceptions
_targetName = _args.targetName if _args.targetName is not None else '_phase2/workdata_test_result.csv'
# listing target directories
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
# output
output = open(_outputname, "w")
output.write("Run,")
# progressing
first = True
progress = tqdm(desc='Collecting data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
if _exptions is not None and item['Run'] in _exptions:
progress.update(1)
progress.set_postfix_str(item['path'])
continue
try:
self.collect_workfile(item, _targetName, output, first)
except Exception as e:
print('Filed to get information: run%02d' % int(item['Run']))
first = False
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
output.close()
def merge_roundtrip_subject(self, _dirpath, _outputname, _args):
# parameter passing
_pattern = _args.pattern if _args.pattern is not None else r'\d+'
_exptions = _args.exceptions
_targetName = _args.targetName if _args.targetName is not None else '_roundtrip'
_nUpdates = _args.nUpdates if _args.nUpdates is not None else 100
_timeQuanta = _args.timeQuanta if _args.timeQuanta is not None else 0.01
_special = _args.special if _args.special is not None else 0
# listing target directories
targets = utils.expandDirs([{
'path': _dirpath
}],
'Run',
_ptn=_pattern,
_sort=True)
self.init_output_files(_outputname)
# progressing
progress = tqdm(desc='Collecting data',
total=len(targets),
unit=' #',
postfix=None)
for item in targets:
if _exptions is not None and item['Run'] in _exptions:
progress.update(1)
progress.set_postfix_str(item['path'])
continue
run = int(item['Run'])
filename = '%s/%s/result.csv' % (item['path'], _targetName)
data = self.load_roundtrip_summary(
filename, 6, _startCol=0) # load 0:5 columns from the file
if data is None:
print("No file of %s" % (filename))
continue
results = data['body']
countDM = 0
numTasks = []
numExecutions = []
sumSizeDM = []
for x in range(0, len(results)):
# WID(0), solutionID(1), DM(2), numTasks(3), numExecutions(4), sumSizeDM(5)
line = results[x]
if line[2] == 0: continue
countDM += 1
numTasks.append(line[3])
numExecutions.append(line[4])
sumSizeDM.append(line[5])
if countDM == 0:
numTasks.append(0)
numExecutions.append(0)
sumSizeDM.append(0)
self.save_roundtrip_summary(_outputname, run, countDM,
len(results), numTasks, numExecutions,
sumSizeDM)
progress.update(1)
progress.set_postfix_str(item['path'])
progress.close()
pass