def myperf_single(vm, col=None, cfg=cfg, benchmarks=benchmarks): assert col, "please provide DB collection to store results into" vmpid = vm.pid RPopen = vm.rpc.root.Popen cg = vm.cg for name, cmd in benchmarks.items(): # start log.notice("launching %s (%s)" % (name, cmd)) p = RPopen(cmd) # p -- benchmark pipe log.notice("warmup sleeping for %s" % cfg.warmup) time.sleep(cfg.warmup) # measurement cg.enable_ipc() cg.get_ipc(reset=True) log.notice("measuring for %s" % cfg.measure) time.sleep(cfg.measure) ipc = cg.get_ipc() stat = {"instructions": ipc, "name": name} log.info(stat) col.save(stat) cg.disable_ipc() # termination assert p.poll() is None, "test unexpectedly terminated" p.killall() wait_idleness(cfg.idfactor * 2)
def start(self): cgroups = [] rpcs = [] self.stop() # just to be sure # START KVMS info("starting kvms") for n in self.cpus: cgkvm = CGKVM(n) time.sleep(0.5) # interval between launching # WAIT TILL THEY START UP idleness = self.idfactor * (len(self.cpus) + 2) self.log.info("waiting for instances, expected idleness is <%s" % idleness) time.sleep(15) wait_idleness(idleness) # CREATE RPC CONNECTIONS info("making connections") for cpu in self.cpus: host = "172.16.5.%s" % (cpu + 1) print("connecting to", host) rpcs += [rpyc.connect(host, port=6666)] for i, cg in enumerate(cgroups): instance = Instance(cg=cg, rpc=rpcs[i], Popen=rpcs[i].root.Popen)
def double(prefix, far=True): topology = CPUTopology() if far: cpu, bgcpu = topology.cpus_no_ht[:2] else: cpu = topology.cpus_no_ht[0] bgcpu = topology.ht_siblings[cpu][0] with cgmgr: vm = cgmgr.start(str(cpu)) bgvm = cgmgr.start(str(bgcpu)) time.sleep(BOOT_TIME) rpc = retry(rpyc.connect, args=(str(vm.addr),), kwargs={"port":6666}, retries=10) bgrpc = retry(rpyc.connect, args=(str(bgvm.addr),), kwargs={"port":6666}, retries=10) RPopen = rpc.root.Popen BGRPopen = bgrpc.root.Popen remains = len(benchmarks)**2 for bgname, bgcmd in benchmarks.items(): log.debug("waiting for idleness") wait_idleness(IDLENESS*3.3) log.warning("launching %s in bg" % bgname) bg = BGRPopen(bgcmd) log.debug("warming up for %s" % WARMUP_TIME) time.sleep(WARMUP_TIME) for name, cmd in benchmarks.items(): print("remains %s tests" % remains) remains -= 1 outdir = s("${prefix}/double/${bgname}/") try: os.makedirs(outdir) except FileExistsError: pass output = outdir + name perf_cmd = PERF_CMD.format(pid=vm.pid, t=MEASURE_TIME, events=events, output=output) log.debug("starting %s" % name) p = RPopen(cmd) log.debug("warming up for %s" % WARMUP_TIME) time.sleep(WARMUP_TIME) log.debug("starting measurements") run(perf_cmd) assert p.poll() is None, "test unexpectedly terminated" assert bg.poll() is None, "bg process suddenly died :(" log.debug("finishing tests") p.killall() gc.collect() bg.killall() time.sleep(1)
def perf_single(vm, col=None, cfg=cfg, benchmarks=benchmarks, events=["cycles"]): assert col, "please provide DB collection to store results into" vmpid = vm.pid RPopen = vm.rpc.root.Popen for name, cmd in benchmarks.items(): # start log.notice("launching %s (%s)" % (name, cmd)) p = RPopen(cmd) # p -- benchmark pipe log.notice("warmup sleeping for %s" % cfg.warmup) time.sleep(cfg.warmup) # measurement stat = perftool.kvmstat(pid=vmpid, events=events, t=cfg.measure, ann=name, norm=True) col.save(stat) log.info(stat) # termination assert p.poll() is None, "test unexpectedly terminated" p.killall() wait_idleness(cfg.idfactor * 2)
def single(vm, prefix): outdir = s("${prefix}/single/") if not exists(outdir): makedirs(outdir) remains = len(basis) for name, cmd in basis.items(): log.debug("remains %s tests" % remains) remains -= 1 outfile = s("${prefix}/single/${name}") log.debug("waiting for idleness") wait_idleness(IDLENESS*2.3) log.debug("starting %s" % name) p = vm.Popen(cmd) log.debug("warming up for %s" % WARMUP_TIME) time.sleep(WARMUP_TIME) log.debug("starting measurements") vm.stat(outfile) assert p.poll() is None, "test unexpectedly terminated" log.debug("finishing test") p.killall() gc.collect()
def measure_double(cg1, cg2, Popen1, Popen2, benchmarks=benchmarks): global warmup global measure assert cg1 and cg2, "I need cgroups to work with" info("measuring performance of TWO tasks") double_ipc = OrderedDict() for b1 in benchmarks: for b2 in benchmarks: wait_idleness() print("starting", b1, b2, end=" ") p1 = Popen1(benchmarks[b1]) time.sleep(warmup) p2 = Popen2(benchmarks[b2]) print(".") cg1.enable_ipc() # cg1.get_ipc(reset=True) cg2.enable_ipc() # cg2.get_ipc(reset=True) # MEASUREMENT time.sleep(measure) ipc1 = cg1.get_ipc() ipc2 = cg2.get_ipc() double_ipc[b1, b2] = (ipc1, ipc2) print(double_ipc) # TERMINATION assert p1.poll() is None, "test unexpectedly terminated on p1" assert p2.poll() is None, "test unexpectedly terminated on p2" p1.killall() p2.killall() cg1.disable_ipc() cg2.disable_ipc() warning("the resulting double ipc:\n", double_ipc) return double_ipc
try: os.kill(args.pid, 0) except ProcessLookupError: raise Exception("no such pid: %s" % args.pid) rpc = rpyc.connect(args.addr, port=6666) RPopen = rpc.root.Popen start, stop, step = args.interval interval = range(start, stop+1, step) for name,cmd in basis.items(): if name not in args.benches: continue wait_idleness(IDLENESS) print("launching %s"%name) if name != "pgbench": cmd = "bencher.py -s 100000 -- "+cmd p = RPopen(cmd) sleep(WARMUP_TIME) for i in interval: print("%s (%s)" % (i, interval)) outdir = "%s/%s/" % (args.outdir, i) os.path.exists(outdir) or os.makedirs(outdir) perfcmd = PERFCMD.format(interval=i, events="instructions:G", output=outdir+name, time=args.time, pid=args.pid) check_call(shlex.split(perfcmd)) assert p.poll() is None, "test unexpectedly terminated"
def main(): parser = argparse.ArgumentParser(description="Run experiments") parser.add_argument("--debug", default=False, const=True, action="store_const", help="enable debug mode") parser.add_argument("-t", "--tests", default=["single", "double", "random", "perf_single"], nargs="*") parser.add_argument("-e", "--events", default=perftool.get_useful_events(), nargs="*") parser.add_argument("--db", required=True, help="name of mongo database") parser.add_argument( "--no-start", type=bool, default=False, help="Assume that instances are already started. Images are not regenerated, \ VMs are not killed on start.", ) parser.add_argument("--idlness", type=bool, const=True, default=False, nargs="?", help="measure idlness and exit") args = parser.parse_args() # INIT if os.geteuid() != 0: sys.exit("you need root to run this scrips") setrlimit(RLIMIT_NOFILE, (10240, 10240)) # signal(SIGCHLD, SIG_IGN) # signal(SIGINT, SIG_IGN) topology = numa.OnlineCPUTopology() log.notice("topology:\n%s" % topology) cpu_name = numa.get_cpu_name() log.debug("cpu name: %s" % cpu_name) # events = perftool.get_useful_events() # log.debug("useful events: %s", events) mongo_client = MongoClient() db = mongo_client[args.db] gc.disable() # MACHINE-SPECIFIC CONFIGURATION hostname = socket.gethostname() if hostname == "fx": run("hdparm -S 0 /dev/sda", sudo="root") cpus_near = [] cpu1 = topology.cpus[0] cpu2 = topology.ht_siblings[cpu1] cpus_near = [cpu1, cpu2] del cpu1, cpu2 cpus_far = topology.cpus_no_ht[:2] cpus_all = topology.cpus # cfg.idfactor = 10 elif hostname == "ux32vd": cpus_near = topology.cpus_no_ht cpus_far = topology.cpus_no_ht cpus_all = topology.cpus_no_ht cfg.idfactor = 3 elif hostname == "p1": cpus_near = topology.cpus_no_ht cpus_far = None cpus_all = cpus_near else: raise Exception("No profile for machine % " % hostname) print("cpus_near:", cpus_near, "cpus_far:", cpus_far) if args.debug: log.critical("Warning! Debug enabled. Using debug database") log.notice(args) cfg.warmup = 0.5 cfg.measure = 0.5 cfg.idfactor *= 10 db = mongo_client[args.db + "_debug"] # PRE-FLIGHT CHECK if not args.no_start: warning("killing all kvms") cgmgr.graceful(timeout=30) subprocess.call("/home/sources/perftests/regen_img.sh") subprocess.check_output("sync") if args.idlness: log.debug("measuring idlness") return print("idleness is", check_idleness()) # EXPERIMENT 1: SINGLE TASK PERFORMANCE (IDEAL PERFORMANCE) if "single" in args.tests: instances = start_instances([cpus_near[0]]) inst = instances[0] measure_single(cg=inst.cg, Popen=inst.Popen) stop_instances(instances) # EXPERIMENT 2: TWO TASK PERFORMANCE # near if "double" in args.tests: instances = start_instances(cpus_near) inst1 = instances[0] inst2 = instances[1] measure_double(cg1=inst1.cg, cg2=inst2.cg, Popen1=inst1.Popen, Popen2=inst2.Popen) stop_instances(instances) # far if cpus_far: instances = start_instances(cpus_far) inst1 = instances[0] inst2 = instances[1] measure_double(cg1=inst1.cg, cg2=inst2.cg, Popen1=inst1.Popen, Popen2=inst2.Popen) stop_instances(instances) # EXPERIMENT 3: arbitrary tests if "random" in args.tests: with VMS(cpus_all) as instances: arbitrary_tests(instances=instances, cpucfg=[1 for _ in cpus_all], num=1000) # EXPERIMENT 4: test with all counters enabled if "perf_single" in args.tests: db.single.drop() col = db.single with RPCMgr("0") as vms: vm = vms["0"] wait_idleness(cfg.idfactor * 2) r = perf_single(vm=vm, cfg=cfg, col=col, events=args.events) # EXPERIMENT 5: measurements stability if "perf_stab" in args.tests: vmname = str(cpus_far[0]) with RPCMgr(vmname) as vms: vm = vms[vmname] log.notice("running tests on VM " + vmname) wait_idleness(cfg.idfactor * 2) for attempt in range(3): for name, evset in evsets.items(): for t in [1, 3, 10, 30, 90, 180, 300]: # for t in [30]: cfg.measure = t if not args.debug else 1 col = db["stab_%s_%ss" % (name, t)] r = perf_single(vm=vm, cfg=cfg, col=col, benchmarks=benchmarks, events=evset) # EXPERIMENT 6: measurements stability if "myperf_stab" in args.tests: vmname = str(cpus_far[0]) with RPCMgr(vmname) as vms: vm = vms[vmname] log.notice("running tests on VM " + vmname) wait_idleness(cfg.idfactor * 2) for attempt in range(3): for t in [1, 3, 10, 30, 90, 180, 300]: # for t in [90]: log.error("gc-collected %s elemets" % gc.collect()) cfg.measure = t if not args.debug else 1 col = db["stab_%ss" % t] r = myperf_single(vm=vm, cfg=cfg, col=col, benchmarks=benchmarks) input("press any key to continue")
def arbitrary_tests(instances=None, cpucfg=[1, 1], num=10, benchmarks=benchmarks): vms = len(instances) result = [] for x in range(num): pipes = [] tstcfg = [] wait_idleness() for i, cnt in enumerate(cpucfg): bs = [] for _ in range(cnt): b = random.choice(list(benchmarks)) bs.append(b) print("on cpu {} we start {}".format(i, b)) Popen = instances[i].Popen pipes += [Popen(benchmarks[b])] time.sleep(warmup / vms * 1.5) # warm-up tstcfg += [bs] for inst in instances: inst.cg.enable_ipc() time.sleep(measure) ipcs = [] for inst in instances: ipcs += [inst.cg.get_ipc()] inst.cg.disable_ipc() for p in pipes: assert p.poll() is None, "test unexpectedly terminated" p.killall() result.append((tstcfg, ipcs)) print(result) return result # self.idfactor = check_idleness(t=3) # self.log.notice("idfactor is %s" % self.idfactor) # assert self.idfactor <= 7, "is the machine really idle?" def start(self): cgroups = [] rpcs = [] self.stop() # just to be sure # START KVMS info("starting kvms") for n in self.cpus: cgkvm = CGKVM(n) time.sleep(0.5) # interval between launching # WAIT TILL THEY START UP idleness = self.idfactor * (len(self.cpus) + 2) self.log.info("waiting for instances, expected idleness is <%s" % idleness) time.sleep(15) wait_idleness(idleness) # CREATE RPC CONNECTIONS info("making connections") for cpu in self.cpus: host = "172.16.5.%s" % (cpu + 1) print("connecting to", host) rpcs += [rpyc.connect(host, port=6666)] for i, cg in enumerate(cgroups): instance = Instance(cg=cg, rpc=rpcs[i], Popen=rpcs[i].root.Popen)