def run(self):
""" Executes the load method while monitoring performance """
config = self.getConfiguration()
# Create and move to the execution path
executionPath = config.get('General', 'ExecutionPath')
if not os.path.isdir(executionPath):
os.makedirs(executionPath)
os.chdir(executionPath)
usageMonitor = config.getboolean('General', 'UsageMonitor')
ioMonitorParam = config.get('General', 'IOMonitor').strip()
ioMonitor = False
ioDevices = None
if ioMonitorParam != '':
ioMonitor = True
ioDevices = ioMonitorParam.split(',')
inputFolder = config.get('Load', 'Folder')
resultsFileAbsPath = os.path.abspath('loading_results')
usageAbsPath = os.path.abspath('loading.usage')
usageImageAbsPath = os.path.abspath('loading_usage.png')
ioAbsPath = None
if ioMonitor:
ioAbsPath = os.path.abspath('loading.io')
ioImageAbsPath = os.path.abspath('loading_io.png')
t0 = time.time()
utils.runMonitor(self.load, (resultsFileAbsPath, ), usageMonitor,
usageAbsPath, ioDevices, ioAbsPath)
totalTime = time.time() - t0
cpu = '-'
mem = '-'
if usageMonitor:
(times, cpus, mems) = utils.parseUsage(usageAbsPath)
utils.saveUsage(times, cpus, mems, 'Loading CPU/MEM',
usageImageAbsPath)
cpu = '%.2f' % cpus.mean()
mem = '%.2f' % mems.mean()
if ioMonitor:
(times, rdata, wdata) = utils.parseIO(ioAbsPath)
utils.saveIO(times, rdata, wdata, 'Loading IO', ioImageAbsPath)
resultsFile = open(resultsFileAbsPath, 'a')
logging.info('Finished loading from' + inputFolder)
size = self.size()
numpoints = 'Total Num. points: ' + str(self.getNumPoints())
logging.info('Total time is: %.2f seconds' % totalTime)
resultsFile.write('total: %.2f \n' % totalTime)
logging.info(size)
resultsFile.write(str(size) + '\n')
logging.info(numpoints)
resultsFile.write(str(numpoints) + '\n')
logging.info('Avg. CPU: ' + cpu + '. Avg. Memory: ' + mem)
resultsFile.write('Avg. CPU: ' + cpu + '. Avg. Memory: \n' + mem)
resultsFile.close()
logging.info('See ' + resultsFileAbsPath)
logging.info('See ' + usageAbsPath)
logging.info('See ' + usageImageAbsPath)
if ioMonitor:
logging.info('See ' + ioAbsPath)
logging.info('See ' + ioImageAbsPath)
def run(self):
""" Executes the load method while monitoring performance """
config = self.getConfiguration()
# Create and move to the execution path
executionPath = config.get("General", "ExecutionPath")
if not os.path.isdir(executionPath):
os.makedirs(executionPath)
os.chdir(executionPath)
usageMonitor = config.getboolean("General", "UsageMonitor")
ioMonitorParam = config.get("General", "IOMonitor").strip()
ioMonitor = False
ioDevices = None
if ioMonitorParam != "":
ioMonitor = True
ioDevices = ioMonitorParam.split(",")
inputFolder = config.get("Load", "Folder")
resultsFileAbsPath = os.path.abspath("loading_results")
usageAbsPath = os.path.abspath("loading.usage")
usageImageAbsPath = os.path.abspath("loading_usage.png")
ioAbsPath = None
if ioMonitor:
ioAbsPath = os.path.abspath("loading.io")
ioImageAbsPath = os.path.abspath("loading_io.png")
t0 = time.time()
utils.runMonitor(self.load, (resultsFileAbsPath,), usageMonitor, usageAbsPath, ioDevices, ioAbsPath)
totalTime = time.time() - t0
cpu = "-"
mem = "-"
if usageMonitor:
(times, cpus, mems) = utils.parseUsage(usageAbsPath)
utils.saveUsage(times, cpus, mems, "Loading CPU/MEM", usageImageAbsPath)
cpu = "%.2f" % cpus.mean()
mem = "%.2f" % mems.mean()
if ioMonitor:
(times, rdata, wdata) = utils.parseIO(ioAbsPath)
utils.saveIO(times, rdata, wdata, "Loading IO", ioImageAbsPath)
resultsFile = open(resultsFileAbsPath, "a")
logging.info("Finished loading from" + inputFolder)
size = self.size()
numpoints = "Total Num. points: " + str(self.getNumPoints())
logging.info("Total time is: %.2f seconds" % totalTime)
resultsFile.write("total: %.2f \n" % totalTime)
logging.info(size)
resultsFile.write(str(size) + "\n")
logging.info(numpoints)
resultsFile.write(str(numpoints) + "\n")
logging.info("Avg. CPU: " + cpu + ". Avg. Memory: " + mem)
resultsFile.write("Avg. CPU: " + cpu + ". Avg. Memory: \n" + mem)
resultsFile.close()
logging.info("See " + resultsFileAbsPath)
logging.info("See " + usageAbsPath)
logging.info("See " + usageImageAbsPath)
if ioMonitor:
logging.info("See " + ioAbsPath)
logging.info("See " + ioImageAbsPath)
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations,
queriesParameters, usageMonitor, ioDevices):
childResultQueue = multiprocessing.Queue()
kill_received = False
while not kill_received:
queryId = None
try:
# This call will patiently wait until new job is available
queryId = tasksQueue.get()
except:
# if there is an error we will quit the generation
kill_received = True
if queryId == None:
# If we receive a None job, it means we can stop this workers
# (all the create-image jobs are done)
kill_received = True
else:
for iterationId in range(numIterations):
queryName = queryId + '_' + '_' + str(iterationId)
usageAbsPath = os.path.abspath(queryName + '.usage')
ioAbsPath = None
if ioDevices != None:
ioAbsPath = os.path.abspath(queryName + '.io')
utils.runMonitor(self.runQuery,
(queryId, iterationId, queriesParameters,
childResultQueue), usageMonitor,
usageAbsPath, ioDevices, ioAbsPath)
[queryId, iterationId, qTime,
qResult] = childResultQueue.get()
(qCPU, qMEM) = (None, None)
if usageMonitor:
(times, cpus, mems) = utils.parseUsage(usageAbsPath)
(qCPU, qMEM) = (cpus.mean(), mems.mean())
imageAbsPath = os.path.abspath(queryName +
'_usage.png')
utils.saveUsage(times, cpus, mems,
queryName + ' CPU/MEM', imageAbsPath)
if ioDevices != None:
(times, rdata, wdata) = utils.parseIO(ioAbsPath)
ioImageAbsPath = os.path.abspath(queryName + '_io.png')
utils.saveIO(times, rdata, wdata, queryName + ' IO',
ioImageAbsPath)
resultsQueue.put((userIndex, queryId, iterationId, qTime,
qResult, qCPU, qMEM))
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices):
childResultQueue = multiprocessing.Queue()
kill_received = False
while not kill_received:
queryId = None
try:
# This call will patiently wait until new job is available
queryId = tasksQueue.get()
except:
# if there is an error we will quit the generation
kill_received = True
if queryId == None:
# If we receive a None job, it means we can stop this workers
# (all the create-image jobs are done)
kill_received = True
else:
for iterationId in range(numIterations):
queryName = queryId + '_' + '_' + str(iterationId)
usageAbsPath = os.path.abspath(queryName + '.usage')
ioAbsPath = None
if ioDevices != None:
ioAbsPath = os.path.abspath(queryName + '.io')
utils.runMonitor(self.runQuery,(queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath)
[queryId, iterationId, qTime, qResult] = childResultQueue.get()
(qCPU,qMEM) = (None, None)
if usageMonitor:
(times, cpus, mems) = utils.parseUsage(usageAbsPath)
(qCPU,qMEM) = (cpus.mean(), mems.mean())
imageAbsPath = os.path.abspath(queryName + '_usage.png')
utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath)
if ioDevices != None:
(times, rdata, wdata) = utils.parseIO(ioAbsPath)
ioImageAbsPath = os.path.abspath(queryName + '_io.png')
utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath)
resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import sys from pointcloud import utils inputFile = sys.argv[1] outputFile = sys.argv[2] (times, cpus, mems) = utils.parseUsage(inputFile) if len(sys.argv) != 3: step = int(sys.argv[3]) times = times[::step] cpus = utils.chunkedMean(cpus, step) mems = utils.chunkedMean(mems, step) utils.saveUsage(times, cpus, mems, "CPU / MEM", outputFile)
all_results[folder]['time'] = fields[1]
total = float(all_results[folder]['time'])
init = float(all_results[folder]['initialize'])
close = float(all_results[folder]['close'])
all_results[folder]['load'] = '%.2f' % (total - (init + close))
if line.count('Size'):
all_results[folder]['totalsize'] = fields[-2]
all_results[folder]['indexsize'] = fields[3]
# if line.count('Avg. CPU:'):
# all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1])
# all_results[folder]['mem'] = fields[-2]
if line.count('Total Num. points:'):
all_results[folder]['npoints'] = fields[-1]
if os.path.isfile(folder + '/loading.usage'):
someUsage = True
all_results[folder]['usage'] = utils.parseUsage(folder +
'/loading.usage')
h = ('Approach', 'Total[s]', 'Init.[s]', 'Load[s]', 'Close[s]', 'Total[MB]',
'Index[MB]', 'Points', 'Points/s', 'Points/MB')
tex = """
\\begin{table}[!ht]
\\centering
\\begin{tabular}{|lrrrrrrrrr|}
\\hline
\multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points} & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\
& Total & Init. & Load & Close & Total & Index & & & \\\\
\\hline
"""
rows = []
for folder in inputFolders:
all_results[folder]['time'] = fields[1]
total = float(all_results[folder]['time'])
init = float(all_results[folder]['initialize'])
close = float(all_results[folder]['close'])
all_results[folder]['load'] = '%.2f' % (total - (init+close))
if line.count('Size'):
all_results[folder]['totalsize'] = fields[-2]
all_results[folder]['indexsize'] = fields[3]
# if line.count('Avg. CPU:'):
# all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1])
# all_results[folder]['mem'] = fields[-2]
if line.count('Total Num. points:'):
all_results[folder]['npoints'] = fields[-1]
if os.path.isfile(folder + '/loading.usage'):
someUsage = True
all_results[folder]['usage'] = utils.parseUsage(folder + '/loading.usage')
h = ('Approach', 'Total[s]', 'Init.[s]' , 'Load[s]', 'Close[s]', 'Total[MB]', 'Index[MB]', 'Points', 'Points/s', 'Points/MB')
tex = """
\\begin{table}[!ht]
\\centering
\\begin{tabular}{|lrrrrrrrrr|}
\\hline
\multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points} & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\
& Total & Init. & Load & Close & Total & Index & & & \\\\
\\hline
"""
rows = []
for folder in inputFolders:
aux = []