示例#1
0
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    r, w = os.pipe()
    forkedPid = os.fork()

    if forkedPid:  # read pickled measurements from the pipe
        os.close(w)
        rPipe = os.fdopen(r)
        r = cPickle.Unpickler(rPipe)
        measurements = r.load()
        rPipe.close()
        os.waitpid(forkedPid, 0)
        return measurements

    else:
        # Sample thread will be destroyed when the forked process _exits
        class Sample(threading.Thread):
            def __init__(self, program):
                threading.Thread.__init__(self)
                self.setDaemon(1)
                self.timedout = False
                self.p = program
                self.maxMem = 0
                self.childpids = None
                self.start()

            def run(self):
                try:
                    remaining = maxtime
                    while remaining > 0:
                        mem = gtop.proc_mem(self.p).resident
                        time.sleep(delay)
                        remaining -= delay
                        # race condition - will child processes have been created yet?
                        self.maxMem = max((mem + self.childmem()) / 1024,
                                          self.maxMem)
                    else:
                        self.timedout = True
                        os.kill(self.p, signal.SIGKILL)
                except OSError, (e, err):
                    if logger: logger.error('%s %s', e, err)

            def childmem(self):
                if self.childpids == None:
                    self.childpids = set()
                    for each in gtop.proclist():
                        if gtop.proc_uid(each).ppid == self.p:
                            self.childpids.add(each)
                mem = 0
                for each in self.childpids:
                    mem += gtop.proc_mem(each).resident
                return mem

        try:

            m = Record(arg)

            # only write pickles to the pipe
            os.close(r)
            wPipe = os.fdopen(w, 'w')
            w = cPickle.Pickler(wPipe)

            # gtop cpu is since machine boot, so we need a before measurement
            cpus0 = gtop.cpu().cpus
            start = time.time()

            # spawn the program in a separate process
            p = Popen(commandline,
                      stdout=outFile,
                      stderr=errFile,
                      stdin=inFile)

            # start a thread to sample the program's resident memory use
            t = Sample(program=p.pid)

            # wait for program exit status and resource usage
            rusage = os.wait3(0)

            # gtop cpu is since machine boot, so we need an after measurement
            elapsed = time.time() - start
            cpus1 = gtop.cpu().cpus

            # summarize measurements
            if t.timedout:
                m.setTimedout()
            elif rusage[1] == os.EX_OK:
                m.setOkay()
            else:
                m.setError()

            m.userSysTime = rusage[2][0] + rusage[2][1]
            m.maxMem = t.maxMem

            load = map(
                lambda t0, t1: int(
                    round(100.0 * (1.0 - float(t1.idle - t0.idle) /
                                   (t1.total - t0.total)))), cpus0, cpus1)

            #load.sort(reverse=1) # maybe more obvious unsorted
            m.cpuLoad = ("% ".join([str(i) for i in load])) + "%"

            m.elapsed = elapsed

        except KeyboardInterrupt:
            os.kill(p.pid, signal.SIGKILL)

        except ZeroDivisionError, (e, err):
            if logger: logger.warn('%s %s', err, 'too fast to measure?')
示例#2
0
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    m = Record(arg)

    # For % CPU usage
    cpu0 = taskManagerCpuTimes()

    ### Use a JobObject so we capture data for child processes as well
    hJob = CreateJobObject(None, 'proctree')

    # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
    freq = LARGE_INTEGER()
    isCounter = windll.kernel32.QueryPerformanceFrequency(byref(freq))
    if isCounter:
        t0 = LARGE_INTEGER()
        t = LARGE_INTEGER()
        windll.kernel32.QueryPerformanceCounter(byref(t0))
    else:  # the number of milliseconds since windows started
        t0 = GetTickCount()

    try:
        # spawn the program in a separate process
        p = Popen(commandline, stdout=outFile, stderr=errFile, stdin=inFile)
        hProcess = int(p._handle)
        AssignProcessToJobObject(hJob, hProcess)

        # wait for program exit status - time out in milliseconds
        waitexit = WaitForSingleObject(hProcess, maxtime * 1000)

        # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
        if isCounter:
            windll.kernel32.QueryPerformanceCounter(byref(t))
            m.elapsed = (t.value - t0.value) / float(freq.value)
        else:  # the number of milliseconds since windows started
            t = GetTickCount()
            m.elapsed = (t - t0) / 1000.0

        if waitexit != 0:
            # terminate any child processes as well
            TerminateJobObject(hJob, -1)
            m.setTimedout()
        elif p.poll() == 0:
            m.setOkay()

            ### Use a JobObject so we capture data for child processes as well
            times = QueryInformationJobObject(
                hJob, JobObjectBasicAccountingInformation)
            #ten million - the number of 100-nanosecond units in one second
            totalusr = times['TotalUserTime'] / nanosecs100
            totalsys = times['TotalKernelTime'] / nanosecs100
            m.userSysTime = totalusr + totalsys

            ### "VM Size" seems the more appropriate measure
            ###
            # corresponds to Peak Mem Usage in Task Manager
            # mem = GetProcessMemoryInfo(hProcess)
            # m.maxMem = mem['PeakWorkingSetSize'] / 1024

            # corresponds to VM Size in Task Manager
            mem = QueryInformationJobObject(hJob,
                                            JobObjectExtendedLimitInformation)
            m.maxMem = mem['PeakJobMemoryUsed'] / 1024

            m.cpuLoad = taskManagerCpuLoad(cpu0, taskManagerCpuTimes(),
                                           totalusr)

        elif p.poll() == 2:
            m.setMissing()
        else:
            m.setError()

    except (OSError, ValueError), (e, err):
        if logger: logger.error('%s %s', e, err)
        m.setError()
示例#3
0
文件: planC.py 项目: ym222bs/shootout
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    r, w = os.pipe()
    forkedPid = os.fork()

    if forkedPid:  # read pickled measurements from the pipe
        os.close(w)
        rPipe = os.fdopen(r)
        r = cPickle.Unpickler(rPipe)
        measurements = r.load()
        rPipe.close()
        os.waitpid(forkedPid, 0)
        return measurements

    else:
        # Sample thread will be destroyed when the forked process _exits
        class Sample(threading.Thread):
            def __init__(self, program):
                threading.Thread.__init__(self)
                self.setDaemon(1)
                self.timedout = False
                self.p = program
                self.maxMem = 0
                self.childpids = None
                self.start()

            def run(self):
                try:
                    remaining = maxtime
                    while remaining > 0:
                        time.sleep(delay)
                        remaining -= delay
                    else:
                        self.timedout = True
                        os.kill(self.p, signal.SIGKILL)
                except OSError, (e, err):
                    if logger: logger.error('%s %s', e, err)

        try:

            m = Record(arg)

            # only write pickles to the pipe
            os.close(r)
            wPipe = os.fdopen(w, 'w')
            w = cPickle.Pickler(wPipe)

            start = time.time()

            # spawn the program in a separate process
            p = Popen(commandline,
                      stdout=outFile,
                      stderr=errFile,
                      stdin=inFile)

            # start a thread to sample the program's resident memory use
            t = Sample(program=p.pid)

            # wait for program exit status and resource usage
            rusage = os.wait3(0)

            elapsed = time.time() - start

            # summarize measurements
            if t.timedout:
                m.setTimedout()
            elif rusage[1] == os.EX_OK:
                m.setOkay()
            else:
                m.setError()

            m.userSysTime = rusage[2][0] + rusage[2][1]
            m.maxMem = t.maxMem
            m.cpuLoad = "%"
            m.elapsed = elapsed

        except KeyboardInterrupt:
            os.kill(p.pid, signal.SIGKILL)

        except ZeroDivisionError, (e, err):
            if logger: logger.warn('%s %s', err, 'too fast to measure?')
示例#4
0
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

      m = Record(arg)

      # For % CPU usage
      cpu0 = taskManagerCpuTimes()

      ### Use a JobObject so we capture data for child processes as well
      hJob = CreateJobObject(None,'proctree')

      # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
      freq = LARGE_INTEGER()
      isCounter = windll.kernel32.QueryPerformanceFrequency(byref(freq))
      if isCounter:
         t0 = LARGE_INTEGER(); t = LARGE_INTEGER()
         windll.kernel32.QueryPerformanceCounter(byref(t0))
      else: # the number of milliseconds since windows started
         t0 = GetTickCount()



      try:
         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile)
         hProcess = int(p._handle)
         AssignProcessToJobObject(hJob,hProcess)

         # wait for program exit status - time out in milliseconds
         waitexit = WaitForSingleObject(hProcess,maxtime*1000)



         # For elapsed time try QueryPerformanceCounter otherwise use GetTickCount
         if isCounter:
            windll.kernel32.QueryPerformanceCounter(byref(t))
            m.elapsed = (t.value - t0.value) / float(freq.value)
         else: # the number of milliseconds since windows started
            t = GetTickCount()
            m.elapsed = (t - t0) / 1000.0



         if waitexit != 0:
            # terminate any child processes as well
            TerminateJobObject(hJob,-1)
            m.setTimedout()
         elif p.poll() == 0:
            m.setOkay()


            ### Use a JobObject so we capture data for child processes as well
            times = QueryInformationJobObject(hJob,JobObjectBasicAccountingInformation)
            #ten million - the number of 100-nanosecond units in one second
            totalusr = times['TotalUserTime'] / nanosecs100
            totalsys = times['TotalKernelTime'] / nanosecs100
            m.userSysTime = totalusr + totalsys


            ### "VM Size" seems the more appropriate measure
            ###
            # corresponds to Peak Mem Usage in Task Manager
            # mem = GetProcessMemoryInfo(hProcess)
            # m.maxMem = mem['PeakWorkingSetSize'] / 1024

            # corresponds to VM Size in Task Manager
            mem = QueryInformationJobObject(hJob,JobObjectExtendedLimitInformation)
            m.maxMem = mem['PeakJobMemoryUsed'] / 1024


            m.cpuLoad = taskManagerCpuLoad(cpu0, taskManagerCpuTimes(), totalusr)

         elif p.poll() == 2:
            m.setMissing()
         else:
            m.setError()



      except (OSError,ValueError), (e,err):
         if logger: logger.error('%s %s',e,err)
         m.setError()
示例#5
0
文件: planA.py 项目: bloff/shootout
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

   r,w = os.pipe()
   forkedPid = os.fork()

   if forkedPid: # read pickled measurements from the pipe
      os.close(w); rPipe = os.fdopen(r); r = cPickle.Unpickler(rPipe)
      measurements = r.load()
      rPipe.close()
      os.waitpid(forkedPid,0)
      return measurements

   else: 
      # Sample thread will be destroyed when the forked process _exits
      class Sample(threading.Thread):

         def __init__(self,program):
            threading.Thread.__init__(self)
            self.setDaemon(1)
            self.timedout = False 
            self.p = program
            self.maxMem = 0
            self.childpids = None   
            self.start() 
 
         def run(self):
            try:              
               remaining = maxtime               
               while remaining > 0: 
                  mem = gtop.proc_mem(self.p).resident                                   
                  time.sleep(delay)                    
                  remaining -= delay
                  # race condition - will child processes have been created yet?
                  self.maxMem = max((mem + self.childmem())/1024, self.maxMem)  
               else:
                  self.timedout = True
                  os.kill(self.p, signal.SIGKILL) 
            except OSError, (e,err):
               if logger: logger.error('%s %s',e,err)

         def childmem(self):
            if self.childpids == None:
               self.childpids = set()
               for each in gtop.proclist():
                  if gtop.proc_uid(each).ppid == self.p:
                     self.childpids.add(each)
            mem = 0
            for each in self.childpids:
               mem += gtop.proc_mem(each).resident
            return mem

       
      try:
         def setAffinity():
            if affinitymask: 
               set_process_affinity_mask(os.getpid(),affinitymask)

         m = Record(arg)

         # only write pickles to the pipe
         os.close(r); wPipe = os.fdopen(w, 'w'); w = cPickle.Pickler(wPipe)

         # gtop cpu is since machine boot, so we need a before measurement
         cpus0 = gtop.cpu().cpus 
         start = time.time()

         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile,preexec_fn=setAffinity)
         
         # start a thread to sample the program's resident memory use
         t = Sample( program = p.pid )

         # wait for program exit status and resource usage
         rusage = os.wait3(0)

         # gtop cpu is since machine boot, so we need an after measurement
         elapsed = time.time() - start
         cpus1 = gtop.cpu().cpus 

         # summarize measurements 
         if t.timedout:
            m.setTimedout()
         elif rusage[1] == os.EX_OK:
            m.setOkay()
         else:
            m.setError()

         m.userSysTime = rusage[2][0] + rusage[2][1]
         m.maxMem = t.maxMem

         load = map( 
            lambda t0,t1: 
               int(round( 
                  100.0 * (1.0 - float(t1.idle-t0.idle)/(t1.total-t0.total))
               ))
            ,cpus0 ,cpus1 )

         #load.sort(reverse=1) # maybe more obvious unsorted
         m.cpuLoad = ("% ".join([str(i) for i in load]))+"%"

         m.elapsed = elapsed


      except KeyboardInterrupt:
         os.kill(p.pid, signal.SIGKILL)

      except ZeroDivisionError, (e,err): 
         if logger: logger.warn('%s %s',err,'too fast to measure?')
示例#6
0
def measure(arg, commandline, delay, maxtime, outFile=None, errFile=None,
            inFile=None, logger=None, affinitymask=None):

   # Monitor :
   # - max memory consumption
   # - user and system time
   # - voluntary and involuntary context switches
   class ProcessMonitor(threading.Thread):

      def __init__(self, process):
         threading.Thread.__init__(self)

         self.process = process
         self.maxmem = 0
         self.time = dict(user=0, system=0)
         self.ctxswitches = dict(voluntary=0, involuntary=0)

         self.setDaemon(1)
         self.start()

      def run(self):
         try:
            while time.time() < start + maxtime:
               if not process.is_running():
                  break

               self.maxmem = max(self.maxmem, self.currentmem())

               self.time['user'] = self.process.get_cpu_times().user
               self.time['system'] = self.process.get_cpu_times().system

               self.ctxswitches['voluntary'] = self.process.get_num_ctx_switches().voluntary
               self.ctxswitches['involuntary'] = self.process.get_num_ctx_switches().involuntary

               time.sleep(delay)
         except psutil.NoSuchProcess:
            pass
         except (OSError, Exception) as e:
            if logger:
               logger.error("%s : %s" % (e.__class__.__name__, e.message))

      def currentmem(self):
         return self.process.get_memory_info().rss \
                  + sum(map(lambda p: p.get_memory_info().rss, self.childrenproc()))

      def childrenproc(self):
         if not self.process.is_running():
            return []
         else:
            return self.process.get_children(recursive=True)

   def set_affinity_mask():
      if sys.platform.startswith("linux") or sys.platform.startswith("win32"):
         if affinitymask:
            proc = psutil.Process(os.getpid())
            cpus = []

            for i in range(psutil.NUM_CPUS):
               if affinitymask & (1 << i) > 0:
                  cpus.append(i)

            proc.set_cpu_affinity(affinitymask)

   def compute_load_per_cpu(cpus0, cpus1):
      cpus = []

      for cpu0, cpu1 in zip(cpus0, cpus1):
         cpus.append(int(round(
            100.0 * (1.0 - float(cpu1.idle - cpu0.idle) / (sum(cpu1._asdict().values()) - sum(cpu0._asdict().values())))
         )))

      return cpus

   try:
      record = Record(arg)

      # psutil cpu is since machine boot, so we need a before measurement
      cpus0 = psutil.cpu_times(percpu=True)
      start = time.time()

      # spawn the program in a separate process
      process = psutil.Popen(commandline,
                             stdout=outFile,
                             stderr=errFile,
                             stdin=inFile,
                             preexec_fn=set_affinity_mask)

      monitor = ProcessMonitor(process)

      # wait for program exit status and resource usage
      timeout = False

      try:
         exitcode = process.wait(timeout=maxtime)
      except psutil.TimeoutExpired:
         timeout = True
         os.kill(process.pid, signal.SIGKILL)

      elapsed = time.time() - start
      cpus1 = psutil.cpu_times(percpu=True)

      # summarize measurements
      if timeout:
         record.setTimedout()
      elif exitcode == os.EX_OK:
         record.setOkay()
      else:
         record.setError()

      record.maxMem = monitor.maxmem / 1024
      record.ctxSwitches = monitor.ctxswitches
      record.cpuLoad = " ".join([str(i) + "%" for i in compute_load_per_cpu(cpus0, cpus1)])

      record.time = dict(user=monitor.time['user'],
                         system=monitor.time['system'],
                         elapsed=elapsed)

   except KeyboardInterrupt:
      os.kill(process.pid, signal.SIGKILL)
   except ZeroDivisionError, (e,err):
      if logger: logger.warn('%s %s',err,'too fast to measure?')
示例#7
0
文件: planC.py 项目: bloff/shootout
def measure(arg,commandline,delay,maxtime,
      outFile=None,errFile=None,inFile=None,logger=None,affinitymask=None):

   r,w = os.pipe()
   forkedPid = os.fork()

   if forkedPid: # read pickled measurements from the pipe
      os.close(w); rPipe = os.fdopen(r); r = cPickle.Unpickler(rPipe)
      measurements = r.load()
      rPipe.close()
      os.waitpid(forkedPid,0)
      return measurements

   else: 
      # Sample thread will be destroyed when the forked process _exits
      class Sample(threading.Thread):

         def __init__(self,program):
            threading.Thread.__init__(self)
            self.setDaemon(1)
            self.timedout = False 
            self.p = program
            self.maxMem = 0
            self.childpids = None   
            self.start() 
 
         def run(self):
            try:              
               remaining = maxtime               
               while remaining > 0:                                   
                  time.sleep(delay)                    
                  remaining -= delay
               else:
                  self.timedout = True
                  os.kill(self.p, signal.SIGKILL) 
            except OSError, (e,err):
               if logger: logger.error('%s %s',e,err)

       
      try:

         m = Record(arg)

         # only write pickles to the pipe
         os.close(r); wPipe = os.fdopen(w, 'w'); w = cPickle.Pickler(wPipe)

         start = time.time()

         # spawn the program in a separate process
         p = Popen(commandline,stdout=outFile,stderr=errFile,stdin=inFile)
         
         # start a thread to sample the program's resident memory use
         t = Sample( program = p.pid )

         # wait for program exit status and resource usage
         rusage = os.wait3(0)


         elapsed = time.time() - start

         # summarize measurements 
         if t.timedout:
            m.setTimedout()
         elif rusage[1] == os.EX_OK:
            m.setOkay()
         else:
            m.setError()

         m.userSysTime = rusage[2][0] + rusage[2][1]
         m.maxMem = t.maxMem
         m.cpuLoad = "%"
         m.elapsed = elapsed


      except KeyboardInterrupt:
         os.kill(p.pid, signal.SIGKILL)

      except ZeroDivisionError, (e,err): 
         if logger: logger.warn('%s %s',err,'too fast to measure?')
def measure(arg,
            commandline,
            delay,
            maxtime,
            outFile=None,
            errFile=None,
            inFile=None,
            logger=None,
            affinitymask=None):

    # Monitor :
    # - max memory consumption
    # - user and system time
    # - voluntary and involuntary context switches
    class ProcessMonitor(threading.Thread):
        def __init__(self, process):
            threading.Thread.__init__(self)

            self.process = process
            self.maxmem = 0
            self.time = dict(user=0, system=0)
            self.ctxswitches = dict(voluntary=0, involuntary=0)

            self.setDaemon(1)
            self.start()

        def run(self):
            try:
                while time.time() < start + maxtime:
                    if not process.is_running():
                        break

                    self.maxmem = max(self.maxmem, self.currentmem())

                    self.time['user'] = self.process.get_cpu_times().user
                    self.time['system'] = self.process.get_cpu_times().system

                    self.ctxswitches[
                        'voluntary'] = self.process.get_num_ctx_switches(
                        ).voluntary
                    self.ctxswitches[
                        'involuntary'] = self.process.get_num_ctx_switches(
                        ).involuntary

                    time.sleep(delay)
            except psutil.NoSuchProcess:
                pass
            except (OSError, Exception) as e:
                if logger:
                    logger.error("%s : %s" % (e.__class__.__name__, e.message))

        def currentmem(self):
            return self.process.get_memory_info().rss \
                     + sum(map(lambda p: p.get_memory_info().rss, self.childrenproc()))

        def childrenproc(self):
            if not self.process.is_running():
                return []
            else:
                return self.process.get_children(recursive=True)

    def set_affinity_mask():
        if sys.platform.startswith("linux") or sys.platform.startswith(
                "win32"):
            if affinitymask:
                proc = psutil.Process(os.getpid())
                cpus = []

                for i in range(psutil.NUM_CPUS):
                    if affinitymask & (1 << i) > 0:
                        cpus.append(i)

                proc.set_cpu_affinity(affinitymask)

    def compute_load_per_cpu(cpus0, cpus1):
        cpus = []

        for cpu0, cpu1 in zip(cpus0, cpus1):
            cpus.append(
                int(
                    round(100.0 * (1.0 - float(cpu1.idle - cpu0.idle) /
                                   (sum(cpu1._asdict().values()) -
                                    sum(cpu0._asdict().values()))))))

        return cpus

    try:
        record = Record(arg)

        # psutil cpu is since machine boot, so we need a before measurement
        cpus0 = psutil.cpu_times(percpu=True)
        start = time.time()

        # spawn the program in a separate process
        process = psutil.Popen(commandline,
                               stdout=outFile,
                               stderr=errFile,
                               stdin=inFile,
                               preexec_fn=set_affinity_mask)

        monitor = ProcessMonitor(process)

        # wait for program exit status and resource usage
        timeout = False

        try:
            exitcode = process.wait(timeout=maxtime)
        except psutil.TimeoutExpired:
            timeout = True
            os.kill(process.pid, signal.SIGKILL)

        elapsed = time.time() - start
        cpus1 = psutil.cpu_times(percpu=True)

        # summarize measurements
        if timeout:
            record.setTimedout()
        elif exitcode == os.EX_OK:
            record.setOkay()
        else:
            record.setError()

        record.maxMem = monitor.maxmem / 1024
        record.ctxSwitches = monitor.ctxswitches
        record.cpuLoad = " ".join(
            [str(i) + "%" for i in compute_load_per_cpu(cpus0, cpus1)])

        record.time = dict(user=monitor.time['user'],
                           system=monitor.time['system'],
                           elapsed=elapsed)

    except KeyboardInterrupt:
        os.kill(process.pid, signal.SIGKILL)
    except ZeroDivisionError, (e, err):
        if logger: logger.warn('%s %s', err, 'too fast to measure?')