Example #1
0
    def run(self):
        """ Executes the load method while monitoring performance """
        config = self.getConfiguration()
        # Create and move to the execution path
        executionPath = config.get('General', 'ExecutionPath')
        if not os.path.isdir(executionPath):
            os.makedirs(executionPath)
        os.chdir(executionPath)
        usageMonitor = config.getboolean('General', 'UsageMonitor')
        ioMonitorParam = config.get('General', 'IOMonitor').strip()
        ioMonitor = False
        ioDevices = None
        if ioMonitorParam != '':
            ioMonitor = True
            ioDevices = ioMonitorParam.split(',')

        inputFolder = config.get('Load', 'Folder')

        resultsFileAbsPath = os.path.abspath('loading_results')
        usageAbsPath = os.path.abspath('loading.usage')
        usageImageAbsPath = os.path.abspath('loading_usage.png')
        ioAbsPath = None
        if ioMonitor:
            ioAbsPath = os.path.abspath('loading.io')
            ioImageAbsPath = os.path.abspath('loading_io.png')

        t0 = time.time()
        utils.runMonitor(self.load, (resultsFileAbsPath, ), usageMonitor,
                         usageAbsPath, ioDevices, ioAbsPath)
        totalTime = time.time() - t0
        cpu = '-'
        mem = '-'
        if usageMonitor:
            (times, cpus, mems) = utils.parseUsage(usageAbsPath)
            utils.saveUsage(times, cpus, mems, 'Loading CPU/MEM',
                            usageImageAbsPath)
            cpu = '%.2f' % cpus.mean()
            mem = '%.2f' % mems.mean()
        if ioMonitor:
            (times, rdata, wdata) = utils.parseIO(ioAbsPath)
            utils.saveIO(times, rdata, wdata, 'Loading IO', ioImageAbsPath)
        resultsFile = open(resultsFileAbsPath, 'a')
        logging.info('Finished loading from' + inputFolder)
        size = self.size()
        numpoints = 'Total Num. points: ' + str(self.getNumPoints())
        logging.info('Total time is: %.2f seconds' % totalTime)
        resultsFile.write('total: %.2f \n' % totalTime)
        logging.info(size)
        resultsFile.write(str(size) + '\n')
        logging.info(numpoints)
        resultsFile.write(str(numpoints) + '\n')
        logging.info('Avg. CPU: ' + cpu + '. Avg. Memory: ' + mem)
        resultsFile.write('Avg. CPU: ' + cpu + '. Avg. Memory: \n' + mem)
        resultsFile.close()
        logging.info('See ' + resultsFileAbsPath)
        logging.info('See ' + usageAbsPath)
        logging.info('See ' + usageImageAbsPath)
        if ioMonitor:
            logging.info('See ' + ioAbsPath)
            logging.info('See ' + ioImageAbsPath)
    def run(self):
        """ Executes the load method while monitoring performance """
        config = self.getConfiguration()
        # Create and move to the execution path
        executionPath = config.get("General", "ExecutionPath")
        if not os.path.isdir(executionPath):
            os.makedirs(executionPath)
        os.chdir(executionPath)
        usageMonitor = config.getboolean("General", "UsageMonitor")
        ioMonitorParam = config.get("General", "IOMonitor").strip()
        ioMonitor = False
        ioDevices = None
        if ioMonitorParam != "":
            ioMonitor = True
            ioDevices = ioMonitorParam.split(",")

        inputFolder = config.get("Load", "Folder")

        resultsFileAbsPath = os.path.abspath("loading_results")
        usageAbsPath = os.path.abspath("loading.usage")
        usageImageAbsPath = os.path.abspath("loading_usage.png")
        ioAbsPath = None
        if ioMonitor:
            ioAbsPath = os.path.abspath("loading.io")
            ioImageAbsPath = os.path.abspath("loading_io.png")

        t0 = time.time()
        utils.runMonitor(self.load, (resultsFileAbsPath,), usageMonitor, usageAbsPath, ioDevices, ioAbsPath)
        totalTime = time.time() - t0
        cpu = "-"
        mem = "-"
        if usageMonitor:
            (times, cpus, mems) = utils.parseUsage(usageAbsPath)
            utils.saveUsage(times, cpus, mems, "Loading CPU/MEM", usageImageAbsPath)
            cpu = "%.2f" % cpus.mean()
            mem = "%.2f" % mems.mean()
        if ioMonitor:
            (times, rdata, wdata) = utils.parseIO(ioAbsPath)
            utils.saveIO(times, rdata, wdata, "Loading IO", ioImageAbsPath)
        resultsFile = open(resultsFileAbsPath, "a")
        logging.info("Finished loading from" + inputFolder)
        size = self.size()
        numpoints = "Total Num. points: " + str(self.getNumPoints())
        logging.info("Total time is: %.2f seconds" % totalTime)
        resultsFile.write("total: %.2f \n" % totalTime)
        logging.info(size)
        resultsFile.write(str(size) + "\n")
        logging.info(numpoints)
        resultsFile.write(str(numpoints) + "\n")
        logging.info("Avg. CPU: " + cpu + ". Avg. Memory: " + mem)
        resultsFile.write("Avg. CPU: " + cpu + ". Avg. Memory: \n" + mem)
        resultsFile.close()
        logging.info("See " + resultsFileAbsPath)
        logging.info("See " + usageAbsPath)
        logging.info("See " + usageImageAbsPath)
        if ioMonitor:
            logging.info("See " + ioAbsPath)
            logging.info("See " + ioImageAbsPath)
Example #3
0
    def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations,
                queriesParameters, usageMonitor, ioDevices):
        childResultQueue = multiprocessing.Queue()
        kill_received = False
        while not kill_received:
            queryId = None
            try:
                # This call will patiently wait until new job is available
                queryId = tasksQueue.get()
            except:
                # if there is an error we will quit the generation
                kill_received = True
            if queryId == None:
                # If we receive a None job, it means we can stop this workers
                # (all the create-image jobs are done)
                kill_received = True
            else:
                for iterationId in range(numIterations):
                    queryName = queryId + '_' + '_' + str(iterationId)
                    usageAbsPath = os.path.abspath(queryName + '.usage')

                    ioAbsPath = None
                    if ioDevices != None:
                        ioAbsPath = os.path.abspath(queryName + '.io')

                    utils.runMonitor(self.runQuery,
                                     (queryId, iterationId, queriesParameters,
                                      childResultQueue), usageMonitor,
                                     usageAbsPath, ioDevices, ioAbsPath)

                    [queryId, iterationId, qTime,
                     qResult] = childResultQueue.get()
                    (qCPU, qMEM) = (None, None)
                    if usageMonitor:
                        (times, cpus, mems) = utils.parseUsage(usageAbsPath)
                        (qCPU, qMEM) = (cpus.mean(), mems.mean())
                        imageAbsPath = os.path.abspath(queryName +
                                                       '_usage.png')
                        utils.saveUsage(times, cpus, mems,
                                        queryName + ' CPU/MEM', imageAbsPath)
                    if ioDevices != None:
                        (times, rdata, wdata) = utils.parseIO(ioAbsPath)
                        ioImageAbsPath = os.path.abspath(queryName + '_io.png')
                        utils.saveIO(times, rdata, wdata, queryName + ' IO',
                                     ioImageAbsPath)

                    resultsQueue.put((userIndex, queryId, iterationId, qTime,
                                      qResult, qCPU, qMEM))
    def runUser(self, userIndex, tasksQueue, resultsQueue, numIterations, queriesParameters, usageMonitor, ioDevices):
        childResultQueue = multiprocessing.Queue()
        kill_received = False
        while not kill_received:
            queryId = None
            try:
                # This call will patiently wait until new job is available
                queryId = tasksQueue.get()
            except:
                # if there is an error we will quit the generation
                kill_received = True
            if queryId == None:
                # If we receive a None job, it means we can stop this workers 
                # (all the create-image jobs are done)
                kill_received = True
            else:            
                for iterationId in range(numIterations):
                    queryName = queryId + '_' + '_' + str(iterationId)
                    usageAbsPath = os.path.abspath(queryName + '.usage')
                    
                    ioAbsPath = None
                    if ioDevices != None:
                        ioAbsPath = os.path.abspath(queryName + '.io')
                        
                    utils.runMonitor(self.runQuery,(queryId, iterationId, queriesParameters, childResultQueue), usageMonitor, usageAbsPath, ioDevices, ioAbsPath)
                    
                    [queryId, iterationId, qTime, qResult] = childResultQueue.get()
                    (qCPU,qMEM) = (None, None)
                    if usageMonitor:
                        (times, cpus, mems) = utils.parseUsage(usageAbsPath)
                        (qCPU,qMEM) = (cpus.mean(), mems.mean())
                        imageAbsPath = os.path.abspath(queryName + '_usage.png')
                        utils.saveUsage(times, cpus, mems, queryName + ' CPU/MEM', imageAbsPath)
                    if ioDevices != None:
                        (times, rdata, wdata) = utils.parseIO(ioAbsPath)
                        ioImageAbsPath = os.path.abspath(queryName + '_io.png')
                        utils.saveIO(times, rdata, wdata, queryName + ' IO', ioImageAbsPath)

                    resultsQueue.put((userIndex, queryId, iterationId, qTime, qResult, qCPU, qMEM))   
#!/usr/bin/env python
################################################################################
#    Created by Oscar Martinez                                                 #
#    [email protected]                                                  #
################################################################################
import sys
from pointcloud import utils

inputFile = sys.argv[1]
outputFile = sys.argv[2]

(times, cpus, mems) = utils.parseUsage(inputFile)

if len(sys.argv) != 3:
    step = int(sys.argv[3])
    times = times[::step]
    cpus = utils.chunkedMean(cpus, step)
    mems = utils.chunkedMean(mems, step)

utils.saveUsage(times, cpus, mems, "CPU / MEM", outputFile)
Example #6
0
                all_results[folder]['time'] = fields[1]
                total = float(all_results[folder]['time'])
                init = float(all_results[folder]['initialize'])
                close = float(all_results[folder]['close'])
                all_results[folder]['load'] = '%.2f' % (total - (init + close))
            if line.count('Size'):
                all_results[folder]['totalsize'] = fields[-2]
                all_results[folder]['indexsize'] = fields[3]
    #        if line.count('Avg. CPU:'):
    #            all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1])
    #            all_results[folder]['mem'] = fields[-2]
            if line.count('Total Num. points:'):
                all_results[folder]['npoints'] = fields[-1]
        if os.path.isfile(folder + '/loading.usage'):
            someUsage = True
            all_results[folder]['usage'] = utils.parseUsage(folder +
                                                            '/loading.usage')

h = ('Approach', 'Total[s]', 'Init.[s]', 'Load[s]', 'Close[s]', 'Total[MB]',
     'Index[MB]', 'Points', 'Points/s', 'Points/MB')

tex = """
\\begin{table}[!ht]
\\centering
\\begin{tabular}{|lrrrrrrrrr|}
\\hline
\multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points}  & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\  
& Total & Init. & Load & Close & Total & Index & & & \\\\ 
\\hline
"""
rows = []
for folder in inputFolders:
                all_results[folder]['time'] = fields[1]
                total = float(all_results[folder]['time'])
                init = float(all_results[folder]['initialize'])
                close = float(all_results[folder]['close'])
                all_results[folder]['load'] = '%.2f' % (total - (init+close))
            if line.count('Size'):
                all_results[folder]['totalsize'] = fields[-2]
                all_results[folder]['indexsize'] = fields[3]
    #        if line.count('Avg. CPU:'):
    #            all_results[folder]['cpu'] = '.'.join(fields[2].split('.')[:-1])
    #            all_results[folder]['mem'] = fields[-2]
            if line.count('Total Num. points:'):
                all_results[folder]['npoints'] = fields[-1]
        if os.path.isfile(folder + '/loading.usage'):
            someUsage = True
            all_results[folder]['usage'] = utils.parseUsage(folder + '/loading.usage')

h = ('Approach', 'Total[s]', 'Init.[s]' , 'Load[s]', 'Close[s]', 'Total[MB]', 'Index[MB]', 'Points', 'Points/s', 'Points/MB')

tex = """
\\begin{table}[!ht]
\\centering
\\begin{tabular}{|lrrrrrrrrr|}
\\hline
\multirow{2}{*}{Approach} & \multicolumn{4}{c}{Time[s]} & \multicolumn{2}{c}{Size[MB]} & \multirow{2}{*}{Points}  & \multirow{2}{*}{Points/s} & \multirow{2}{*}{Points/MB}\\\\  
& Total & Init. & Load & Close & Total & Index & & & \\\\ 
\\hline
"""
rows = []
for folder in inputFolders:
    aux = []