def run(self): """ Executes the load method while monitoring performance """ config = self.getConfiguration() # Create and move to the execution path executionPath = config.get('General', 'ExecutionPath') if not os.path.isdir(executionPath): os.makedirs(executionPath) os.chdir(executionPath) usageMonitor = config.getboolean('General', 'UsageMonitor') ioMonitorParam = config.get('General', 'IOMonitor').strip() ioMonitor = False ioDevices = None if ioMonitorParam != '': ioMonitor = True ioDevices = ioMonitorParam.split(',') inputFolder = config.get('Load', 'Folder') resultsFileAbsPath = os.path.abspath('loading_results') usageAbsPath = os.path.abspath('loading.usage') usageImageAbsPath = os.path.abspath('loading_usage.png') ioAbsPath = None if ioMonitor: ioAbsPath = os.path.abspath('loading.io') ioImageAbsPath = os.path.abspath('loading_io.png') t0 = time.time() utils.runMonitor(self.load, (resultsFileAbsPath, ), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) totalTime = time.time() - t0 cpu = '-' mem = '-' if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) utils.saveUsage(times, cpus, mems, 'Loading CPU/MEM', usageImageAbsPath) cpu = '%.2f' % cpus.mean() mem = '%.2f' % mems.mean() if ioMonitor: (times, rdata, wdata) = utils.parseIO(ioAbsPath) utils.saveIO(times, rdata, wdata, 'Loading IO', ioImageAbsPath) resultsFile = open(resultsFileAbsPath, 'a') logging.info('Finished loading from' + inputFolder) size = self.size() numpoints = 'Total Num. points: ' + str(self.getNumPoints()) logging.info('Total time is: %.2f seconds' % totalTime) resultsFile.write('total: %.2f \n' % totalTime) logging.info(size) resultsFile.write(str(size) + '\n') logging.info(numpoints) resultsFile.write(str(numpoints) + '\n') logging.info('Avg. CPU: ' + cpu + '. Avg. Memory: ' + mem) resultsFile.write('Avg. CPU: ' + cpu + '. Avg. Memory: \n' + mem) resultsFile.close() logging.info('See ' + resultsFileAbsPath) logging.info('See ' + usageAbsPath) logging.info('See ' + usageImageAbsPath) if ioMonitor: logging.info('See ' + ioAbsPath) logging.info('See ' + ioImageAbsPath)
def run(self): """ Executes the load method while monitoring performance """ config = self.getConfiguration() # Create and move to the execution path executionPath = config.get("General", "ExecutionPath") if not os.path.isdir(executionPath): os.makedirs(executionPath) os.chdir(executionPath) usageMonitor = config.getboolean("General", "UsageMonitor") ioMonitorParam = config.get("General", "IOMonitor").strip() ioMonitor = False ioDevices = None if ioMonitorParam != "": ioMonitor = True ioDevices = ioMonitorParam.split(",") inputFolder = config.get("Load", "Folder") resultsFileAbsPath = os.path.abspath("loading_results") usageAbsPath = os.path.abspath("loading.usage") usageImageAbsPath = os.path.abspath("loading_usage.png") ioAbsPath = None if ioMonitor: ioAbsPath = os.path.abspath("loading.io") ioImageAbsPath = os.path.abspath("loading_io.png") t0 = time.time() utils.runMonitor(self.load, (resultsFileAbsPath,), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) totalTime = time.time() - t0 cpu = "-" mem = "-" if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) utils.saveUsage(times, cpus, mems, "Loading CPU/MEM", usageImageAbsPath) cpu = "%.2f" % cpus.mean() mem = "%.2f" % mems.mean() if ioMonitor: (times, rdata, wdata) = utils.parseIO(ioAbsPath) utils.saveIO(times, rdata, wdata, "Loading IO", ioImageAbsPath) resultsFile = open(resultsFileAbsPath, "a") logging.info("Finished loading from" + inputFolder) size = self.size() numpoints = "Total Num. points: " + str(self.getNumPoints()) logging.info("Total time is: %.2f seconds" % totalTime) resultsFile.write("total: %.2f \n" % totalTime) logging.info(size) resultsFile.write(str(size) + "\n") logging.info(numpoints) resultsFile.write(str(numpoints) + "\n") logging.info("Avg. CPU: " + cpu + ". Avg. Memory: " + mem) resultsFile.write("Avg. CPU: " + cpu + ". Avg. Memory: \n" + mem) resultsFile.close() logging.info("See " + resultsFileAbsPath) logging.info("See " + usageAbsPath) logging.info("See " + usageImageAbsPath) if ioMonitor: logging.info("See " + ioAbsPath) logging.info("See " + ioImageAbsPath)
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices): childResultQueue = multiprocessing.Queue() kill_received = False while not kill_received: queryId = None try: # This call will patiently wait until new job is available queryId = tasksQueue.get() except: # if there is an error we will quit the generation kill_received = True if queryId == None: # If we receive a None job, it means we can stop this workers # (all the create-image jobs are done) kill_received = True else: for iterationId in range(numIterations): queryName = queryId + '_' + '_' + str(iterationId) usageAbsPath = os.path.abspath(queryName + '.usage') ioAbsPath = None if ioDevices != None: ioAbsPath = os.path.abspath(queryName + '.io') utils.runMonitor(self.runQuery, (queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) [queryId, iterationId, qTime, qResult] = childResultQueue.get() (qCPU, qMEM) = (None, None) if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) (qCPU, qMEM) = (cpus.mean(), mems.mean()) imageAbsPath = os.path.abspath(queryName + '_usage.png') utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath) if ioDevices != None: (times, rdata, wdata) = utils.parseIO(ioAbsPath) ioImageAbsPath = os.path.abspath(queryName + '_io.png') utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath) resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices): childResultQueue = multiprocessing.Queue() kill_received = False while not kill_received: queryId = None try: # This call will patiently wait until new job is available queryId = tasksQueue.get() except: # if there is an error we will quit the generation kill_received = True if queryId == None: # If we receive a None job, it means we can stop this workers # (all the create-image jobs are done) kill_received = True else: for iterationId in range(numIterations): queryName = queryId + '_' + '_' + str(iterationId) usageAbsPath = os.path.abspath(queryName + '.usage') ioAbsPath = None if ioDevices != None: ioAbsPath = os.path.abspath(queryName + '.io') utils.runMonitor(self.runQuery,(queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) [queryId, iterationId, qTime, qResult] = childResultQueue.get() (qCPU,qMEM) = (None, None) if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) (qCPU,qMEM) = (cpus.mean(), mems.mean()) imageAbsPath = os.path.abspath(queryName + '_usage.png') utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath) if ioDevices != None: (times, rdata, wdata) = utils.parseIO(ioAbsPath) ioImageAbsPath = os.path.abspath(queryName + '_io.png') utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath) resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import sys from pointcloud import utils inputFile = sys.argv[1] outputFile = sys.argv[2] (times, cpus, mems) = utils.parseUsage(inputFile) if len(sys.argv) != 3: step = int(sys.argv[3]) times = times[::step] cpus = utils.chunkedMean(cpus, step) mems = utils.chunkedMean(mems, step) utils.saveUsage(times, cpus, mems, "CPU / MEM", outputFile)
all_results[folder]['time'] = fields[1] total = float(all_results[folder]['time']) init = float(all_results[folder]['initialize']) close = float(all_results[folder]['close']) all_results[folder]['load'] = '%.2f' % (total - (init + close)) if line.count('Size'): all_results[folder]['totalsize'] = fields[-2] all_results[folder]['indexsize'] = fields[3] # if line.count('Avg. CPU:'): # all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1]) # all_results[folder]['mem'] = fields[-2] if line.count('Total Num. points:'): all_results[folder]['npoints'] = fields[-1] if os.path.isfile(folder + '/loading.usage'): someUsage = True all_results[folder]['usage'] = utils.parseUsage(folder + '/loading.usage') h = ('Approach', 'Total[s]', 'Init.[s]', 'Load[s]', 'Close[s]', 'Total[MB]', 'Index[MB]', 'Points', 'Points/s', 'Points/MB') tex = """ \\begin{table}[!ht] \\centering \\begin{tabular}{|lrrrrrrrrr|} \\hline \multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points} & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\ & Total & Init. & Load & Close & Total & Index & & & \\\\ \\hline """ rows = [] for folder in inputFolders:
all_results[folder]['time'] = fields[1] total = float(all_results[folder]['time']) init = float(all_results[folder]['initialize']) close = float(all_results[folder]['close']) all_results[folder]['load'] = '%.2f' % (total - (init+close)) if line.count('Size'): all_results[folder]['totalsize'] = fields[-2] all_results[folder]['indexsize'] = fields[3] # if line.count('Avg. CPU:'): # all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1]) # all_results[folder]['mem'] = fields[-2] if line.count('Total Num. points:'): all_results[folder]['npoints'] = fields[-1] if os.path.isfile(folder + '/loading.usage'): someUsage = True all_results[folder]['usage'] = utils.parseUsage(folder + '/loading.usage') h = ('Approach', 'Total[s]', 'Init.[s]' , 'Load[s]', 'Close[s]', 'Total[MB]', 'Index[MB]', 'Points', 'Points/s', 'Points/MB') tex = """ \\begin{table}[!ht] \\centering \\begin{tabular}{|lrrrrrrrrr|} \\hline \multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points} & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\ & Total & Init. & Load & Close & Total & Index & & & \\\\ \\hline """ rows = [] for folder in inputFolders: aux = []