def run(self): """ Executes the load method while monitoring performance """ config = self.getConfiguration() # Create and move to the execution path executionPath = config.get('General', 'ExecutionPath') if not os.path.isdir(executionPath): os.makedirs(executionPath) os.chdir(executionPath) usageMonitor = config.getboolean('General', 'UsageMonitor') ioMonitorParam = config.get('General', 'IOMonitor').strip() ioMonitor = False ioDevices = None if ioMonitorParam != '': ioMonitor = True ioDevices = ioMonitorParam.split(',') inputFolder = config.get('Load', 'Folder') resultsFileAbsPath = os.path.abspath('loading_results') usageAbsPath = os.path.abspath('loading.usage') usageImageAbsPath = os.path.abspath('loading_usage.png') ioAbsPath = None if ioMonitor: ioAbsPath = os.path.abspath('loading.io') ioImageAbsPath = os.path.abspath('loading_io.png') t0 = time.time() utils.runMonitor(self.load, (resultsFileAbsPath, ), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) totalTime = time.time() - t0 cpu = '-' mem = '-' if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) utils.saveUsage(times, cpus, mems, 'Loading CPU/MEM', usageImageAbsPath) cpu = '%.2f' % cpus.mean() mem = '%.2f' % mems.mean() if ioMonitor: (times, rdata, wdata) = utils.parseIO(ioAbsPath) utils.saveIO(times, rdata, wdata, 'Loading IO', ioImageAbsPath) resultsFile = open(resultsFileAbsPath, 'a') logging.info('Finished loading from' + inputFolder) size = self.size() numpoints = 'Total Num. points: ' + str(self.getNumPoints()) logging.info('Total time is: %.2f seconds' % totalTime) resultsFile.write('total: %.2f \n' % totalTime) logging.info(size) resultsFile.write(str(size) + '\n') logging.info(numpoints) resultsFile.write(str(numpoints) + '\n') logging.info('Avg. CPU: ' + cpu + '. Avg. Memory: ' + mem) resultsFile.write('Avg. CPU: ' + cpu + '. Avg. Memory: \n' + mem) resultsFile.close() logging.info('See ' + resultsFileAbsPath) logging.info('See ' + usageAbsPath) logging.info('See ' + usageImageAbsPath) if ioMonitor: logging.info('See ' + ioAbsPath) logging.info('See ' + ioImageAbsPath)
def run(self): """ Executes the load method while monitoring performance """ config = self.getConfiguration() # Create and move to the execution path executionPath = config.get("General", "ExecutionPath") if not os.path.isdir(executionPath): os.makedirs(executionPath) os.chdir(executionPath) usageMonitor = config.getboolean("General", "UsageMonitor") ioMonitorParam = config.get("General", "IOMonitor").strip() ioMonitor = False ioDevices = None if ioMonitorParam != "": ioMonitor = True ioDevices = ioMonitorParam.split(",") inputFolder = config.get("Load", "Folder") resultsFileAbsPath = os.path.abspath("loading_results") usageAbsPath = os.path.abspath("loading.usage") usageImageAbsPath = os.path.abspath("loading_usage.png") ioAbsPath = None if ioMonitor: ioAbsPath = os.path.abspath("loading.io") ioImageAbsPath = os.path.abspath("loading_io.png") t0 = time.time() utils.runMonitor(self.load, (resultsFileAbsPath,), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) totalTime = time.time() - t0 cpu = "-" mem = "-" if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) utils.saveUsage(times, cpus, mems, "Loading CPU/MEM", usageImageAbsPath) cpu = "%.2f" % cpus.mean() mem = "%.2f" % mems.mean() if ioMonitor: (times, rdata, wdata) = utils.parseIO(ioAbsPath) utils.saveIO(times, rdata, wdata, "Loading IO", ioImageAbsPath) resultsFile = open(resultsFileAbsPath, "a") logging.info("Finished loading from" + inputFolder) size = self.size() numpoints = "Total Num. points: " + str(self.getNumPoints()) logging.info("Total time is: %.2f seconds" % totalTime) resultsFile.write("total: %.2f \n" % totalTime) logging.info(size) resultsFile.write(str(size) + "\n") logging.info(numpoints) resultsFile.write(str(numpoints) + "\n") logging.info("Avg. CPU: " + cpu + ". Avg. Memory: " + mem) resultsFile.write("Avg. CPU: " + cpu + ". Avg. Memory: \n" + mem) resultsFile.close() logging.info("See " + resultsFileAbsPath) logging.info("See " + usageAbsPath) logging.info("See " + usageImageAbsPath) if ioMonitor: logging.info("See " + ioAbsPath) logging.info("See " + ioImageAbsPath)
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices): childResultQueue = multiprocessing.Queue() kill_received = False while not kill_received: queryId = None try: # This call will patiently wait until new job is available queryId = tasksQueue.get() except: # if there is an error we will quit the generation kill_received = True if queryId == None: # If we receive a None job, it means we can stop this workers # (all the create-image jobs are done) kill_received = True else: for iterationId in range(numIterations): queryName = queryId + '_' + '_' + str(iterationId) usageAbsPath = os.path.abspath(queryName + '.usage') ioAbsPath = None if ioDevices != None: ioAbsPath = os.path.abspath(queryName + '.io') utils.runMonitor(self.runQuery, (queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) [queryId, iterationId, qTime, qResult] = childResultQueue.get() (qCPU, qMEM) = (None, None) if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) (qCPU, qMEM) = (cpus.mean(), mems.mean()) imageAbsPath = os.path.abspath(queryName + '_usage.png') utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath) if ioDevices != None: (times, rdata, wdata) = utils.parseIO(ioAbsPath) ioImageAbsPath = os.path.abspath(queryName + '_io.png') utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath) resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))
def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices): childResultQueue = multiprocessing.Queue() kill_received = False while not kill_received: queryId = None try: # This call will patiently wait until new job is available queryId = tasksQueue.get() except: # if there is an error we will quit the generation kill_received = True if queryId == None: # If we receive a None job, it means we can stop this workers # (all the create-image jobs are done) kill_received = True else: for iterationId in range(numIterations): queryName = queryId + '_' + '_' + str(iterationId) usageAbsPath = os.path.abspath(queryName + '.usage') ioAbsPath = None if ioDevices != None: ioAbsPath = os.path.abspath(queryName + '.io') utils.runMonitor(self.runQuery,(queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath) [queryId, iterationId, qTime, qResult] = childResultQueue.get() (qCPU,qMEM) = (None, None) if usageMonitor: (times, cpus, mems) = utils.parseUsage(usageAbsPath) (qCPU,qMEM) = (cpus.mean(), mems.mean()) imageAbsPath = os.path.abspath(queryName + '_usage.png') utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath) if ioDevices != None: (times, rdata, wdata) = utils.parseIO(ioAbsPath) ioImageAbsPath = os.path.abspath(queryName + '_io.png') utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath) resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import sys from pointcloud import utils inputFile = sys.argv[1] outputFile = sys.argv[2] (times, rdata, wdata) = utils.parseIO(inputFile) if len(sys.argv) != 3: step = int(sys.argv[3]) times = times[::step] for k in rdata: rdata[k] = utils.chunkedMean(rdata[k], step) wdata[k] = utils.chunkedMean(wdata[k], step) utils.saveIO(times, rdata, wdata, 'IO', outputFile)
#!/usr/bin/env python ################################################################################ # Created by Oscar Martinez # # [email protected] # ################################################################################ import sys from pointcloud import utils inputFile = sys.argv[1] outputFile = sys.argv[2] (times, rdata, wdata) = utils.parseIO(inputFile) if len(sys.argv) != 3: step = int(sys.argv[3]) times = times[::step] for k in rdata: rdata[k] = utils.chunkedMean(rdata[k],step) wdata[k] = utils.chunkedMean(wdata[k],step) utils.saveIO(times, rdata, wdata, 'IO', outputFile)