Exemplo n.º 1
0
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    r, w = os.pipe()
    forkedPid = os.fork()

    if forkedPid:  # read pickled measurements from the pipe
        os.close(w)
        rPipe = os.fdopen(r)
        r = cPickle.Unpickler(rPipe)
        measurements = r.load()
        rPipe.close()
        os.waitpid(forkedPid, 0)
        return measurements

    else:
        # Sample thread will be destroyed when the forked process _exits
        class Sample(threading.Thread):
            def __init__(self, program):
                threading.Thread.__init__(self)
                self.setDaemon(1)
                self.timedout = False
                self.p = program
                self.maxMem = 0
                self.childpids = None
                self.start()

            def run(self):
                try:
                    remaining = maxtime
                    while remaining > 0:
                        mem = gtop.proc_mem(self.p).resident
                        time.sleep(delay)
                        remaining -= delay
                        # race condition - will child processes have been created yet?
                        self.maxMem = max((mem + self.childmem()) / 1024,
                                          self.maxMem)
                    else:
                        self.timedout = True
                        os.kill(self.p, signal.SIGKILL)
                except OSError, (e, err):
                    if logger: logger.error('%s %s', e, err)

            def childmem(self):
                if self.childpids == None:
                    self.childpids = set()
                    for each in gtop.proclist():
                        if gtop.proc_uid(each).ppid == self.p:
                            self.childpids.add(each)
                mem = 0
                for each in self.childpids:
                    mem += gtop.proc_mem(each).resident
                return mem

        try:

            m = Record(arg)

            # only write pickles to the pipe
            os.close(r)
            wPipe = os.fdopen(w, 'w')
            w = cPickle.Pickler(wPipe)

            # gtop cpu is since machine boot, so we need a before measurement
            cpus0 = gtop.cpu().cpus
            start = time.time()

            # spawn the program in a separate process
            p = Popen(commandline,
                      stdout=outFile,
                      stderr=errFile,
                      stdin=inFile)

            # start a thread to sample the program's resident memory use
            t = Sample(program=p.pid)

            # wait for program exit status and resource usage
            rusage = os.wait3(0)

            # gtop cpu is since machine boot, so we need an after measurement
            elapsed = time.time() - start
            cpus1 = gtop.cpu().cpus

            # summarize measurements
            if t.timedout:
                m.setTimedout()
            elif rusage[1] == os.EX_OK:
                m.setOkay()
            else:
                m.setError()

            m.userSysTime = rusage[2][0] + rusage[2][1]
            m.maxMem = t.maxMem

            load = map(
                lambda t0, t1: int(
                    round(100.0 * (1.0 - float(t1.idle - t0.idle) /
                                   (t1.total - t0.total)))), cpus0, cpus1)

            #load.sort(reverse=1) # maybe more obvious unsorted
            m.cpuLoad = ("% ".join([str(i) for i in load])) + "%"

            m.elapsed = elapsed

        except KeyboardInterrupt:
            os.kill(p.pid, signal.SIGKILL)

        except ZeroDivisionError, (e, err):
            if logger: logger.warn('%s %s', err, 'too fast to measure?')
Exemplo n.º 2
0
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    r, w = os.pipe()
    forkedPid = os.fork()

    if forkedPid:  # read pickled measurements from the pipe
        os.close(w)
        rPipe = os.fdopen(r)
        r = cPickle.Unpickler(rPipe)
        measurements = r.load()
        rPipe.close()
        os.waitpid(forkedPid, 0)
        return measurements

    else:
        # Sample thread will be destroyed when the forked process _exits
        class Sample(threading.Thread):
            def __init__(self, program):
                threading.Thread.__init__(self)
                self.setDaemon(1)
                self.timedout = False
                self.p = program
                self.maxMem = 0
                self.childpids = None
                self.start()

            def run(self):
                try:
                    remaining = maxtime
                    while remaining > 0:
                        time.sleep(delay)
                        remaining -= delay
                    else:
                        self.timedout = True
                        os.kill(self.p, signal.SIGKILL)
                except OSError, (e, err):
                    if logger: logger.error('%s %s', e, err)

        try:

            m = Record(arg)

            # only write pickles to the pipe
            os.close(r)
            wPipe = os.fdopen(w, 'w')
            w = cPickle.Pickler(wPipe)

            start = time.time()

            # spawn the program in a separate process
            p = Popen(commandline,
                      stdout=outFile,
                      stderr=errFile,
                      stdin=inFile)

            # start a thread to sample the program's resident memory use
            t = Sample(program=p.pid)

            # wait for program exit status and resource usage
            rusage = os.wait3(0)

            elapsed = time.time() - start

            # summarize measurements
            if t.timedout:
                m.setTimedout()
            elif rusage[1] == os.EX_OK:
                m.setOkay()
            else:
                m.setError()

            m.userSysTime = rusage[2][0] + rusage[2][1]
            m.maxMem = t.maxMem
            m.cpuLoad = "%"
            m.elapsed = elapsed

        except KeyboardInterrupt:
            os.kill(p.pid, signal.SIGKILL)

        except ZeroDivisionError, (e, err):
            if logger: logger.warn('%s %s', err, 'too fast to measure?')
Exemplo n.º 3
0
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    m = Record(arg)

    # For % CPU usage
    cpu0 = taskManagerCpuTimes()

    ### Use a JobObject so we capture data for child processes as well
    hJob = CreateJobObject(None, 'proctree')

    # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
    freq = LARGE_INTEGER()
    isCounter = windll.kernel32.QueryPerformanceFrequency(byref(freq))
    if isCounter:
        t0 = LARGE_INTEGER()
        t = LARGE_INTEGER()
        windll.kernel32.QueryPerformanceCounter(byref(t0))
    else:  # the number of milliseconds since windows started
        t0 = GetTickCount()

    try:
        # spawn the program in a separate process
        p = Popen(commandline, stdout=outFile, stderr=errFile, stdin=inFile)
        hProcess = int(p._handle)
        AssignProcessToJobObject(hJob, hProcess)

        # wait for program exit status - time out in milliseconds
        waitexit = WaitForSingleObject(hProcess, maxtime * 1000)

        # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
        if isCounter:
            windll.kernel32.QueryPerformanceCounter(byref(t))
            m.elapsed = (t.value - t0.value) / float(freq.value)
        else:  # the number of milliseconds since windows started
            t = GetTickCount()
            m.elapsed = (t - t0) / 1000.0

        if waitexit != 0:
            # terminate any child processes as well
            TerminateJobObject(hJob, -1)
            m.setTimedout()
        elif p.poll() == 0:
            m.setOkay()

            ### Use a JobObject so we capture data for child processes as well
            times = QueryInformationJobObject(
                hJob, JobObjectBasicAccountingInformation)
            #ten million - the number of 100-nanosecond units in one second
            totalusr = times['TotalUserTime'] / nanosecs100
            totalsys = times['TotalKernelTime'] / nanosecs100
            m.userSysTime = totalusr + totalsys

            ### "VM Size" seems the more appropriate measure
            ###
            # corresponds to Peak Mem Usage in Task Manager
            # mem = GetProcessMemoryInfo(hProcess)
            # m.maxMem = mem['PeakWorkingSetSize'] / 1024

            # corresponds to VM Size in Task Manager
            mem = QueryInformationJobObject(hJob,
                                            JobObjectExtendedLimitInformation)
            m.maxMem = mem['PeakJobMemoryUsed'] / 1024

            m.cpuLoad = taskManagerCpuLoad(cpu0, taskManagerCpuTimes(),
                                           totalusr)

        elif p.poll() == 2:
            m.setMissing()
        else:
            m.setError()

    except (OSError, ValueError), (e, err):
        if logger: logger.error('%s %s', e, err)
        m.setError()
Exemplo n.º 4
0
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

      m = Record(arg)

      # For % CPU usage
      cpu0 = taskManagerCpuTimes()

      ### Use a JobObject so we capture data for child processes as well
      hJob = CreateJobObject(None,'proctree')

      # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
      freq = LARGE_INTEGER()
      isCounter = windll.kernel32.QueryPerformanceFrequency(byref(freq))
      if isCounter:
         t0 = LARGE_INTEGER(); t = LARGE_INTEGER()
         windll.kernel32.QueryPerformanceCounter(byref(t0))
      else: # the number of milliseconds since windows started
         t0 = GetTickCount()



      try:
         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile)
         hProcess = int(p._handle)
         AssignProcessToJobObject(hJob,hProcess)

         # wait for program exit status - time out in milliseconds
         waitexit = WaitForSingleObject(hProcess,maxtime*1000)



         # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
         if isCounter:
            windll.kernel32.QueryPerformanceCounter(byref(t))
            m.elapsed = (t.value - t0.value) / float(freq.value)
         else: # the number of milliseconds since windows started
            t = GetTickCount()
            m.elapsed = (t - t0) / 1000.0



         if waitexit != 0:
            # terminate any child processes as well
            TerminateJobObject(hJob,-1)
            m.setTimedout()
         elif p.poll() == 0:
            m.setOkay()


            ### Use a JobObject so we capture data for child processes as well
            times = QueryInformationJobObject(hJob,JobObjectBasicAccountingInformation)
            #ten million - the number of 100-nanosecond units in one second
            totalusr = times['TotalUserTime'] / nanosecs100
            totalsys = times['TotalKernelTime'] / nanosecs100
            m.userSysTime = totalusr + totalsys


            ### "VM Size" seems the more appropriate measure
            ###
            # corresponds to Peak Mem Usage in Task Manager
            # mem = GetProcessMemoryInfo(hProcess)
            # m.maxMem = mem['PeakWorkingSetSize'] / 1024

            # corresponds to VM Size in Task Manager
            mem = QueryInformationJobObject(hJob,JobObjectExtendedLimitInformation)
            m.maxMem = mem['PeakJobMemoryUsed'] / 1024


            m.cpuLoad = taskManagerCpuLoad(cpu0, taskManagerCpuTimes(), totalusr)

         elif p.poll() == 2:
            m.setMissing()
         else:
            m.setError()



      except (OSError,ValueError), (e,err):
         if logger: logger.error('%s %s',e,err)
         m.setError()
Exemplo n.º 5
0
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

   r,w = os.pipe()
   forkedPid = os.fork()

   if forkedPid: # read pickled measurements from the pipe
      os.close(w); rPipe = os.fdopen(r); r = cPickle.Unpickler(rPipe)
      measurements = r.load()
      rPipe.close()
      os.waitpid(forkedPid,0)
      return measurements

   else: 
      # Sample thread will be destroyed when the forked process _exits
      class Sample(threading.Thread):

         def __init__(self,program):
            threading.Thread.__init__(self)
            self.setDaemon(1)
            self.timedout = False 
            self.p = program
            self.maxMem = 0
            self.childpids = None   
            self.start() 
 
         def run(self):
            try:              
               remaining = maxtime               
               while remaining > 0: 
                  mem = gtop.proc_mem(self.p).resident                                   
                  time.sleep(delay)                    
                  remaining -= delay
                  # race condition - will child processes have been created yet?
                  self.maxMem = max((mem + self.childmem())/1024, self.maxMem)  
               else:
                  self.timedout = True
                  os.kill(self.p, signal.SIGKILL) 
            except OSError, (e,err):
               if logger: logger.error('%s %s',e,err)

         def childmem(self):
            if self.childpids == None:
               self.childpids = set()
               for each in gtop.proclist():
                  if gtop.proc_uid(each).ppid == self.p:
                     self.childpids.add(each)
            mem = 0
            for each in self.childpids:
               mem += gtop.proc_mem(each).resident
            return mem

       
      try:
         def setAffinity():
            if affinitymask: 
               set_process_affinity_mask(os.getpid(),affinitymask)

         m = Record(arg)

         # only write pickles to the pipe
         os.close(r); wPipe = os.fdopen(w, 'w'); w = cPickle.Pickler(wPipe)

         # gtop cpu is since machine boot, so we need a before measurement
         cpus0 = gtop.cpu().cpus 
         start = time.time()

         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile,preexec_fn=setAffinity)
         
         # start a thread to sample the program's resident memory use
         t = Sample( program = p.pid )

         # wait for program exit status and resource usage
         rusage = os.wait3(0)

         # gtop cpu is since machine boot, so we need an after measurement
         elapsed = time.time() - start
         cpus1 = gtop.cpu().cpus 

         # summarize measurements 
         if t.timedout:
            m.setTimedout()
         elif rusage[1] == os.EX_OK:
            m.setOkay()
         else:
            m.setError()

         m.userSysTime = rusage[2][0] + rusage[2][1]
         m.maxMem = t.maxMem

         load = map( 
            lambda t0,t1: 
               int(round( 
                  100.0 * (1.0 - float(t1.idle-t0.idle)/(t1.total-t0.total))
               ))
            ,cpus0 ,cpus1 )

         #load.sort(reverse=1) # maybe more obvious unsorted
         m.cpuLoad = ("% ".join([str(i) for i in load]))+"%"

         m.elapsed = elapsed


      except KeyboardInterrupt:
         os.kill(p.pid, signal.SIGKILL)

      except ZeroDivisionError, (e,err): 
         if logger: logger.warn('%s %s',err,'too fast to measure?')
Exemplo n.º 6
0
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

   r,w = os.pipe()
   forkedPid = os.fork()

   if forkedPid: # read pickled measurements from the pipe
      os.close(w); rPipe = os.fdopen(r); r = cPickle.Unpickler(rPipe)
      measurements = r.load()
      rPipe.close()
      os.waitpid(forkedPid,0)
      return measurements

   else: 
      # Sample thread will be destroyed when the forked process _exits
      class Sample(threading.Thread):

         def __init__(self,program):
            threading.Thread.__init__(self)
            self.setDaemon(1)
            self.timedout = False 
            self.p = program
            self.maxMem = 0
            self.childpids = None   
            self.start() 
 
         def run(self):
            try:              
               remaining = maxtime               
               while remaining > 0:                                   
                  time.sleep(delay)                    
                  remaining -= delay
               else:
                  self.timedout = True
                  os.kill(self.p, signal.SIGKILL) 
            except OSError, (e,err):
               if logger: logger.error('%s %s',e,err)

       
      try:

         m = Record(arg)

         # only write pickles to the pipe
         os.close(r); wPipe = os.fdopen(w, 'w'); w = cPickle.Pickler(wPipe)

         start = time.time()

         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile)
         
         # start a thread to sample the program's resident memory use
         t = Sample( program = p.pid )

         # wait for program exit status and resource usage
         rusage = os.wait3(0)


         elapsed = time.time() - start

         # summarize measurements 
         if t.timedout:
            m.setTimedout()
         elif rusage[1] == os.EX_OK:
            m.setOkay()
         else:
            m.setError()

         m.userSysTime = rusage[2][0] + rusage[2][1]
         m.maxMem = t.maxMem
         m.cpuLoad = "%"
         m.elapsed = elapsed


      except KeyboardInterrupt:
         os.kill(p.pid, signal.SIGKILL)

      except ZeroDivisionError, (e,err): 
         if logger: logger.warn('%s %s',err,'too fast to measure?')