Пример #1
0
 def stop(self):
     self.hlist.killall("memcached mcperf")
     self.hlist.stop_trafgen()
     self.hlist.stop_mpstat()
     self.log(T.colored("... Waiting for qdiscs to drain", "blue"))
     progress(5)
     self.hlist.remove_qdiscs()
     self.hlist.rmmod_qfq()
     self.hlist.rmmod_eyeq()
     if self.opts("sniffer"):
         self.hsniffer.copy_local(e('', tmpdir=config['SNIFFER_TMPDIR']),
                                 self.opts("exptid") + "-snf",
                                 tmpdir=config['SNIFFER_TMPDIR'])
     self.hlist.copy_by_host(e('logs'), self.opts("outdir") + "/logs",
                             self.opts("exptid"))
     self.hlist.clear_intel_hw_rate_limits(config['NIC_HW_QUEUES'])
     self.hlist.clear_mellanox_hw_rate_limits()
Пример #2
0
    def start_hadoop(self):
        self.master = Host("10.0.1.20")
        self.master.cmd("cd /usr/local/hadoop/conf;" +
                        "cp slaves.tenant1 slaves; cp masters.tenant1 masters;")
        self.start_dnsd()
        self.master.cmd("start-all.sh")
        progress(120)

        dir = self.opts("dir")
        cmd = "mkdir -p %s; cd /usr/local/hadoop; " % dir
        cmd += "hadoop jar hadoop-examples-0.20.205.0.jar "

        self.sid = self.get_sort_id()
        out = os.path.join(dir, "hadoop-progress.txt")
        cmd += "sort random-data sorted-data-%s " % self.sid
        cmd += " > %s 2>&1; " % out
        self.hadoop_job = self.master.cmd_async(cmd)
Пример #3
0
 def stop(self):
     if self.opts("create"):
         return
     if self.opts("destroy"):
         self.hlist.remove_tenants()
         return
     try:
         while 1:
             done = self.check_hadoop_done()
             if done:
                 break
             else:
                 print "Waiting for hadoop job...", datetime.datetime.now()
                 progress(240)
             continue
     except Exception, e:
         print "Hadoop job not found", e
Пример #4
0
 def stop(self):
     self.hlist.killall("memcached mcperf")
     self.hlist.stop_trafgen()
     self.hlist.stop_mpstat()
     self.log(T.colored("... Waiting for qdiscs to drain", "blue"))
     progress(5)
     self.hlist.remove_qdiscs()
     self.hlist.rmmod_qfq()
     self.hlist.rmmod_eyeq()
     if self.opts("sniffer"):
         self.hsniffer.copy_local(e('', tmpdir=config['SNIFFER_TMPDIR']),
                                  self.opts("exptid") + "-snf",
                                  tmpdir=config['SNIFFER_TMPDIR'])
     self.hlist.copy_by_host(e('logs'),
                             self.opts("outdir") + "/logs",
                             self.opts("exptid"))
     self.hlist.clear_intel_hw_rate_limits(config['NIC_HW_QUEUES'])
     self.hlist.clear_mellanox_hw_rate_limits()
Пример #5
0
    def start_loadgen(self, out="loadgen.txt", tid=2, traffic=None):
        dir = self.opts("dir")
        out = os.path.join(dir, out)
        LOADGEN = "/root/vimal/exports/loadgen"
        # Start in all hosts
        if traffic is None:
            return
        port = 12345 + tid
        for h in self.hlist.lst:
            ip = h.get_tenant_ip(tid)
            cmd = "mkdir -p %s; " % dir
            cmd += "%s -i %s -vv " % (LOADGEN, ip)
            cmd += " -l %s -p 1000000 -f %s > %s" % (port, traffic, out)
            h.cmd_async(cmd)

        print "Waiting for loadgen to start..."
        progress(20)
        for h in self.hlist.lst:
            ip = h.get_tenant_ip(tid)
            p = Popen("nc -nzv %s %s" % (ip, port), shell=True)
        return
Пример #6
0
    def start_pa_process(self, out="paggr.txt", tid=1, cpu=None, P=1):
        dir = self.opts("dir")
        out = os.path.join(dir, out)
        CLIENT = "/root/vimal/exports/incast_app/l25_tcp_client"
        SERVER = "/root/vimal/exports/incast_app/l25_tcp_server"
        if cpu is None:
            cpu = self.nextcpu
            self.nextcpu = (self.nextcpu + 2) % 8
        for h in self.hlist.lst:
            ip = h.get_tenant_ip(tid)
            cmd = "mkdir -p %s; " % dir
            if args.bind:
                cmd += "taskset -c %s,%s  " % (cpu, cpu+1)
            cmd += " %s %s" % (SERVER, ip)
            h.cmd_async(cmd)

        print "Waiting for servers to start..."
        progress(1)
        print "starting client (tid=%s) on %s..." % (tid, host_ips[0])
        h0 = Host(host_ips[0])

        # Generate input file for client
        size = self.opts("size")
        inpfile = "~/vimal/exports/incast_app/input/get%s_P%s_tenant%s.dat" % (size, P, tid)
        cmd = "python tests/genconfig.py --traffic paggr -n 15 "
        cmd += "-P %s --size %s --repeat %s --tenant %s " % (P, size, self.opts("repeat"), tid)
        cmd += "> %s" % inpfile
        Popen(cmd, shell=True).wait()

        dir = self.opts("dir")
        outfile = os.path.join(dir, out)

        cmd = 'mkdir -p %s; ' % dir
        if args.bind:
            cmd += "taskset -c %s,%s  " % (cpu, cpu+1)
        cmd += " %s %s > %s" % (CLIENT, inpfile, outfile)
        h0.cmd_async(cmd)
        return
Пример #7
0
 def stop(self):
     if self.opts("create"):
         return
     if self.opts("destroy"):
         self.hlist.remove_tenants()
         return
     start = datetime.datetime.now()
     try:
         while 1:
             done = True
             for i in xrange(self.opts("nhadoop")):
                 done = done and self.check_hadoop_done(i)
             if done:
                 break
             else:
                 print "Waiting for hadoop job(s)   start: ", start
                 print "        now: ", datetime.datetime.now()
                 try:
                     progress(60)
                 except:
                     break
             continue
     except Exception, e:
         print "Hadoop job not found", e
Пример #8
0
    def start_loadgen(self, out="loadgen.txt", tid=2, traffic=None, cpu=None):
        dir = self.opts("dir")
        out = os.path.join(dir, out)
        LOADGEN = "/root/vimal/exports/loadgen"
        # Start in all hosts
        if traffic is None:
            return
        if cpu is None:
            cpu = self.nextcpu
            self.nextcpu = (self.nextcpu + 2) % 8
        port = 12345 + tid
        for h in self.hlist.lst:
            ip = h.get_tenant_ip(tid)
            cmd = "mkdir -p %s; " % dir
            if args.pin:
                cmd += "taskset -c %s,%s  " % (cpu, cpu+1)
            cmd += " %s -i %s -vv " % (LOADGEN, ip)
            cmd += " -l %s -p 1000000 -f %s > %s" % (port, traffic, out)
            h.cmd_async(cmd)

        print "Waiting for loadgen to start..."
        progress(5)
        for h in self.hlist.lst:
            ip = h.get_tenant_ip(tid)

            while 1:
                p = Popen("nc -nzv %s %s" % (ip, port),
                          shell=True, stdout=PIPE, stderr=PIPE)
                stdout, stderr = p.communicate()
                if 'open' in stderr:
                    print T.colored("started %s:%s" % (ip, port), "green")
                    break
                else:
                    print T.colored("  retrying %s:%s" % (ip, port), "yellow")
                    sleep(0.5)
        return
Пример #9
0
    def start(self):
        sniffer = self.opts("sniffer")

        hservers = HostList()
        hclients = HostList()
        hlist = HostList()
        if sniffer:
            hsniffer = Host(sniffer)

        self.log(T.colored("Servers:---", "green"))
        for ip in self.opts("servers"):
            h = Host(ip)
            hservers.append(h)
            hlist.append(h)
            self.log(T.colored(ip, "green"))

        self.log(T.colored("Clients:---", "yellow"))
        for ip in self.opts("clients"):
            h = Host(ip)
            hclients.append(h)
            hlist.append(h)
            self.log(T.colored(ip, "yellow"))

        # Reset/clear state on servers and clients
        hlist.rmrf(e(""))
        hlist.mkdir(e("logs"))

        # Log the servers and clients used for the experiment
        local_cmd("mkdir -p %s/logs" % self.opts("outdir"))
        hostsfile = "%s/logs/hostsfile.txt" % self.opts("outdir")
        hostsfd = open(hostsfile, 'w')
        hostsfd.write("Servers:\n")
        for ip in self.opts("servers"):
            hostsfd.write("  " + ip + "\n")
        hostsfd.write("Clients:\n")
        for ip in self.opts("clients"):
            hostsfd.write("  " + ip + "\n")
        hostsfd.close()

        if sniffer:
            hsniffer.rmrf(e("", tmpdir=config['SNIFFER_TMPDIR']))
            hsniffer.mkdir(e("logs", tmpdir=config['SNIFFER_TMPDIR']))
            hsniffer.cmd("killall -9 %s" % config['SNIFFER'])

        hlist.rmmod()
        hlist.stop_trafgen()
        hlist.cmd("sudo service memcached stop")
        hlist.killall("udp memcached mcperf")
        hlist.remove_qdiscs()
        hlist.clear_intel_hw_rate_limits(config['NIC_HW_QUEUES'])
        hlist.clear_mellanox_hw_rate_limits()
        sleep(1)

        hlist.configure_tcp_limit_output_bytes()

        # Find available CPU cores for all tenants
        start_port = self.opts("startport")
        avail_cpus = [ x for x in xrange(0, config['NUM_CPUS'])
                             if x not in config['EXCLUDE_CPUS'] ]

        # Setup interrupt affinity
        # Configure interrupts to only be sent to respective CPU cores
        # to which the tenants are pinned
        if self.opts("mctenants") + 2*self.opts("trafgentenants") > len(avail_cpus):
            tenant_cpus = avail_cpus
            self.log(T.colored("WARNING: Multiple tenants sharing CPU cores", "red"))
        else:
            tenant_cpus = avail_cpus[:self.opts("mctenants") + 2 * self.opts("trafgentenants")]

        hlist.configure_iface_interrupt_affinity(tenant_cpus)

        # NOTE: Tenant to CPU cores mapping:
        # -- First 'mctenants' CPU cores are used to pin memcached or mcperf
        #    instances on servers and clients respectively.
        # -- Next 'trafgentenants' CPU cores are used to bind trafgen sink
        #    processes for each tenant on each host
        #    Last 'trafgentenants' CPU cores are used to bind trafgen generator
        #    processes for each tenant on each host
        #    So, each trafgen tenant requires 2 CPU cores (1 for sink and 1 for
        #    generator)

        # Start memcached on servers - one instance for each tenant, pinned to a
        # different CPU core
        assigned_cpus = 0
        for tenant in xrange(0, self.opts("mctenants")):
            self.start_memcached(hservers, mem = 2048,
                                 port = start_port + tenant,
                                 threads = 1,
                                 cpus = [avail_cpus[assigned_cpus %
                                                    len(avail_cpus)]])
            assigned_cpus += 1

        # Start trafgen servers/sinks - one instance for each tenant, pinned to
        # a different CPU core on each host
        for tenant in xrange(0, self.opts("trafgentenants")):
            tenant_id = "%d_%d" % (tenant, self.opts("trafgentenants"))
            self.start_trafgen_server(hlist, proto=self.opts("trafgenproto"),
                                      tenant_id = tenant_id,
                                      port = start_port + 1000 + tenant,
                                      cpus = [avail_cpus[assigned_cpus %
                                                         len(avail_cpus)]],
                                      dir=e('logs'))
            assigned_cpus += 1

        # If mcworkload=get, first run mcperf with set requests to full up the
        # cache.  For each (tenant, server) pair, create a separate mcperf
        # instance on each client.
        if self.opts("mcworkload") == "get":

            tmp_assigned_cpus = 0
            hlist.mkdir(e("logs_unused"))
            for tenant in xrange(0, self.opts("mctenants")):
                for (srv_id, hserver) in enumerate(hservers.lst):
                    server_ip = socket.gethostbyname(hserver.hostname())
                    for (cli_id, hclient) in enumerate(hclients.lst):

                        # Index of tenant and client connecting to this
                        # particular server for this tenant.
                        tenant_id = "%d_%d" % (tenant, self.opts("mctenants"))
                        client_id = "%d_%d" % (cli_id, len(hclients.lst))

                        # We use the traffic class number as the seed for the
                        # random number generator so that all instances
                        # generate unique random number sequences.
                        seed = (start_port +
                                (tenant * len(hservers.lst) * len(hclients.lst)) +
                                (srv_id * len(hclients.lst)) +
                                (cli_id))

                        self.start_mcperf(hclient, server_ip,
                                          tenant_id, client_id,
                                          seed = seed,
                                          port = start_port + tenant,
                                          time = MC_PREPOPULATE_TIME,
                                          nconn = self.opts("mcnconn"),
                                          mcrate = 5000,
                                          mcexp = self.opts("mcexp"),
                                          workload = "set",
                                          mcsize = self.opts("mcsize"),
                                          cpus = [avail_cpus[tmp_assigned_cpus %
                                                             len(avail_cpus)]],
                                          dir=e('logs_unused'))
                tmp_assigned_cpus += 1

            self.log(T.colored("Populating caches first", "blue"))
            progress(MC_PREPOPULATE_TIME + 5)

        # Configure rate limits
        # mcperf tenants:
        # On server, configure separate rate limit to each mctenant's client
        # On client, configure separate rate limit to each mctenant's server
        # trafgen tenants:
        # On each host, configure separate rate limits for traffic to each
        # other host, for each trafgentenant
        trafgen_pair_rate = (0 if not self.opts("trafgentenants") else
                             (self.opts("trafgen_total_rate") * 1.0 /
                              (self.opts("trafgentenants") * (len(hlist.lst) - 1))))
        # Client to server traffic
        mc_pair_rate_client = (0 if not self.opts("mctenants") else
                               (self.opts("mc_total_rate_client") * 1.0 /
                                (self.opts("mctenants") * len(hservers.lst))))
        # Server to client traffic
        mc_pair_rate_server = (0 if not self.opts("mctenants") else
                               (self.opts("mc_total_rate_server") * 1.0 /
                                (self.opts("mctenants") * len(hclients.lst))))

        self.log(T.colored("Pair rate mc client = %s" % mc_pair_rate_client, "blue"))
        self.log(T.colored("Pair rate mc server = %s" % mc_pair_rate_server, "blue"))
        self.log(T.colored("Trafgen pair rate = %s" % trafgen_pair_rate, "blue"))
        if self.opts("rl") == "htb":
            hlist.mc_add_htb_qdisc(self.opts("htb_mtu"))
        elif self.opts("rl") == "qfq":
            hlist.mc_add_qfq_qdisc(self.opts("mtu"))
        elif self.opts("rl") == "eyeq":
            hlist.mc_add_htb_qdisc(self.opts("mtu"), True)

        # Qdisc classes
        # class 1 : default class
        # Separate class for each (mctenant, srv_id, cli_id) tuple
        # (start_port +
        #  (tenant * num_servers * num_clients) +
        #  (srv_id * num_clients) +
        #  (cli_id)) :  On client, this represents traffic to srv_id for tenant
        #               On server, this represents traffic to cli_id for tenant
        # Separate class for each (trafgentenant, hsrc, hdst) tuple
        # (start_port + 10000 +
        #  (tenant * num_hosts * (num_hosts - 1)) +
        #  (src_id * (num_hosts - 1)) +
        #  (dst_id)) :  Traffic from src_host to dst_host for each tenant

        for tenant in xrange(0, self.opts("mctenants")):

            for (srv_id, hserver) in enumerate(hservers.lst):
                server_ip = socket.gethostbyname(hserver.hostname())

                for (cli_id, hclient) in enumerate(hclients.lst):
                    client_ip = socket.gethostbyname(hclient.hostname())

                    srv_port = start_port + tenant
                    rate_str_client = '%.3fMbit' % mc_pair_rate_client
                    rate_str_server = '%.3fMbit' % mc_pair_rate_server
                    klass = (start_port +
                             (tenant * len(hservers.lst) * len(hclients.lst)) +
                             (srv_id * len(hclients.lst)) +
                             (cli_id))

                    if self.opts("rl") in ["htb", "eyeq"]:
                        hclient.mc_add_htb_class(rate=rate_str_client,
                                                 ceil=rate_str_client,
                                                 klass=klass,
                                                 htb_mtu=self.opts("htb_mtu"))
                        hserver.mc_add_htb_class(rate=rate_str_server,
                                                 ceil=rate_str_server,
                                                 klass=klass,
                                                 htb_mtu=self.opts("htb_mtu"))
                    elif self.opts("rl") == "qfq":
                        hclient.mc_add_qfq_class(rate=mc_pair_rate_client,
                                                 klass=klass,
                                                 mtu=self.opts("mtu"))
                        hserver.mc_add_qfq_class(rate=mc_pair_rate_server,
                                                 klass=klass,
                                                 mtu=self.opts("mtu"))

                    if self.opts("rl") in ["htb", "eyeq", "qfq"]:
                        # Client -> Server traffic
                        hclient.mc_add_qdisc_filter(server_ip, sport=0,
                                                    dport=srv_port, klass=klass)
                        # Server -> Client traffic
                        hserver.mc_add_qdisc_filter(client_ip, sport=srv_port,
                                                    dport=0, klass=klass)

        for tenant in xrange(0, self.opts("trafgentenants")):
            trafgen_port = start_port + 1000 + tenant
            for (src_id, hsrc) in enumerate(hlist.lst):
                src_ip = socket.gethostbyname(hsrc.hostname())
                for (dst_id, hdst) in enumerate(hlist.lst):
                    if hsrc == hdst:
                        continue

                    # NOTE: Some klass ID's would be unused (when src_id ==
                    # dst_id).
                    # NOTE: Trafgen server -> client traffic (only ACKs) is
                    # allocated a rate limit of only 5Mbit.
                    dst_ip = socket.gethostbyname(hdst.hostname())
                    rate_str = '%.3fMbit' % trafgen_pair_rate
                    rate_str_acks = '5Mbit'
                    klass = (start_port + 10000 +
                             (tenant * len(hlist.lst) * len(hlist.lst)) +
                             (src_id * len(hlist.lst)) +
                             (dst_id))

                    if self.opts("rl") in ["htb", "eyeq"]:
                        hsrc.mc_add_htb_class(rate=rate_str, ceil=rate_str,
                                              klass=klass,
                                              htb_mtu=self.opts("htb_mtu"))
                        hdst.mc_add_htb_class(rate=rate_str_acks,
                                              ceil=rate_str_acks, klass=klass,
                                              htb_mtu=self.opts("htb_mtu"))
                    elif self.opts("rl") == "qfq":
                        hsrc.mc_add_qfq_class(rate=trafgen_pair_rate,
                                              klass=klass, mtu=self.opts("mtu"))
                        hdst.mc_add_qfq_class(rate=5, klass=klass,
                                              mtu=self.opts("mtu"))

                    if self.opts("rl") in ["htb", "eyeq", "qfq"]:
                        # Trafgen client -> server traffic filter
                        hsrc.mc_add_qdisc_filter(dst_ip, sport=0,
                                                 dport=trafgen_port, klass=klass)
                        # Trafgen server -> client traffic filter (only affects ACKs)
                        hdst.mc_add_qdisc_filter(src_ip, sport=trafgen_port,
                                                 dport=0, klass=klass)


        hlist.start_bw_monitor(e('logs'))
        hlist.start_mpstat(e('logs'))
        hlist.start_perf_monitor(e('logs'), self.opts("t"))
        hlist.set_mtu(self.opts("mtu"))
        if sniffer:
            hsniffer.start_sniffer_delayed(e('logs', tmpdir=config['SNIFFER_TMPDIR']),
                    board=0, delay=config['SNIFFER_DELAY'],
                    duration=config['SNIFFER_DURATION'])
        sleep(1)

        # Start trafgen clients to generate background all-to-all traffic.
        # For each (tenant, destination) pair, create a separate trafgen
        # instance on the source host. This is required since trafgen currently
        # only supports a single destination per instance.
        for tenant in xrange(0, self.opts("trafgentenants")):
            tenant_id = "%d_%d" % (tenant, self.opts("trafgentenants"))
            for hsrc in hlist.lst:
                for hdst in hlist.lst:
                    if hsrc == hdst:
                        continue
                    dst_ip = socket.gethostbyname(hdst.hostname())

                    self.start_trafgen_client(hsrc, dst_ip,
                                              tenant_id = tenant_id,
                                              proto=self.opts("trafgenproto"),
                                              port = start_port + 1000 + tenant,
                                              mtu = self.opts("mtu"),
                                              cpus = [avail_cpus[assigned_cpus %
                                                      len(avail_cpus)]],
                                              dir = e('logs'))
            assigned_cpus += 1

        # Start mcperf clients to generate requests. For each (tenant, server)
        # pair, create a separate mcperf instance. This is required since mcperf
        # does not have an option to send requests randomly to the available
        # memcached servers.
        tmp_assigned_cpus = 0
        for tenant in xrange(0, self.opts("mctenants")):
            for hserver in hservers.lst:
                server_ip = socket.gethostbyname(hserver.hostname())
                for (cli_id, hclient) in enumerate(hclients.lst):

                    # Index of tenant and client connecting to this particular
                    # server for this tenant.
                    tenant_id = "%d_%d" % (tenant, self.opts("mctenants"))
                    client_id = "%d_%d" % (cli_id, len(hclients.lst))

                    # We use traffic class number as the seed for the
                    # random number generator so that all instances
                    # generate unique random number sequences. But we want to
                    # use a different seed that that used for the initial "set"
                    # workload for populating caches. So we increment the
                    # respective class numbers by say 11213 to get the seed.
                    seed = (start_port + 11213 +
                            (tenant * len(hservers.lst) * len(hclients.lst)) +
                            (srv_id * len(hclients.lst)) +
                            (cli_id))

                    self.start_mcperf(hclient, server_ip, tenant_id, client_id,
                                      seed = seed,
                                      port = start_port + tenant,
                                      time = self.opts("t"),
                                      nconn = self.opts("mcnconn"),
                                      mcrate = self.opts("mcrate"),
                                      mcexp = self.opts("mcexp"),
                                      workload = self.opts("mcworkload"),
                                      mcsize = self.opts("mcsize"),
                                      cpus = [avail_cpus[tmp_assigned_cpus %
                                                         len(avail_cpus)]],
                                      dir=e('logs'))
            tmp_assigned_cpus += 1

        self.hservers = hservers
        self.hclients = hclients
        self.hlist = hlist
        if sniffer:
            self.hsniffer = hsniffer
Пример #10
0
    def start(self):
        sniffer = self.opts("sniffer")

        hservers = HostList()
        hclients = HostList()
        hlist = HostList()
        if sniffer:
            hsniffer = Host(sniffer)

        self.log(T.colored("Servers:---", "green"))
        for ip in self.opts("servers"):
            h = Host(ip)
            hservers.append(h)
            hlist.append(h)
            self.log(T.colored(ip, "green"))

        self.log(T.colored("Clients:---", "yellow"))
        for ip in self.opts("clients"):
            h = Host(ip)
            hclients.append(h)
            hlist.append(h)
            self.log(T.colored(ip, "yellow"))

        # Reset/clear state on servers and clients
        hlist.rmrf(e(""))
        hlist.mkdir(e("logs"))

        # Log the servers and clients used for the experiment
        local_cmd("mkdir -p %s/logs" % self.opts("outdir"))
        hostsfile = "%s/logs/hostsfile.txt" % self.opts("outdir")
        hostsfd = open(hostsfile, 'w')
        hostsfd.write("Servers:\n")
        for ip in self.opts("servers"):
            hostsfd.write("  " + ip + "\n")
        hostsfd.write("Clients:\n")
        for ip in self.opts("clients"):
            hostsfd.write("  " + ip + "\n")
        hostsfd.close()

        if sniffer:
            hsniffer.rmrf(e("", tmpdir=config['SNIFFER_TMPDIR']))
            hsniffer.mkdir(e("logs", tmpdir=config['SNIFFER_TMPDIR']))
            hsniffer.cmd("killall -9 %s" % config['SNIFFER'])

        hlist.rmmod()
        hlist.stop_trafgen()
        hlist.cmd("sudo service memcached stop")
        hlist.killall("udp memcached mcperf")
        hlist.remove_qdiscs()
        hlist.clear_intel_hw_rate_limits(config['NIC_HW_QUEUES'])
        hlist.clear_mellanox_hw_rate_limits()
        sleep(1)

        hlist.configure_tcp_limit_output_bytes()

        # Find available CPU cores for all tenants
        start_port = self.opts("startport")
        avail_cpus = [
            x for x in xrange(0, config['NUM_CPUS'])
            if x not in config['EXCLUDE_CPUS']
        ]

        # Setup interrupt affinity
        # Configure interrupts to only be sent to respective CPU cores
        # to which the tenants are pinned
        if self.opts("mctenants") + 2 * self.opts("trafgentenants") > len(
                avail_cpus):
            tenant_cpus = avail_cpus
            self.log(
                T.colored("WARNING: Multiple tenants sharing CPU cores",
                          "red"))
        else:
            tenant_cpus = avail_cpus[:self.opts("mctenants") +
                                     2 * self.opts("trafgentenants")]

        hlist.configure_iface_interrupt_affinity(tenant_cpus)

        # NOTE: Tenant to CPU cores mapping:
        # -- First 'mctenants' CPU cores are used to pin memcached or mcperf
        #    instances on servers and clients respectively.
        # -- Next 'trafgentenants' CPU cores are used to bind trafgen sink
        #    processes for each tenant on each host
        #    Last 'trafgentenants' CPU cores are used to bind trafgen generator
        #    processes for each tenant on each host
        #    So, each trafgen tenant requires 2 CPU cores (1 for sink and 1 for
        #    generator)

        # Start memcached on servers - one instance for each tenant, pinned to a
        # different CPU core
        assigned_cpus = 0
        for tenant in xrange(0, self.opts("mctenants")):
            self.start_memcached(
                hservers,
                mem=2048,
                port=start_port + tenant,
                threads=1,
                cpus=[avail_cpus[assigned_cpus % len(avail_cpus)]])
            assigned_cpus += 1

        # Start trafgen servers/sinks - one instance for each tenant, pinned to
        # a different CPU core on each host
        for tenant in xrange(0, self.opts("trafgentenants")):
            tenant_id = "%d_%d" % (tenant, self.opts("trafgentenants"))
            self.start_trafgen_server(
                hlist,
                proto=self.opts("trafgenproto"),
                tenant_id=tenant_id,
                port=start_port + 1000 + tenant,
                cpus=[avail_cpus[assigned_cpus % len(avail_cpus)]],
                dir=e('logs'))
            assigned_cpus += 1

        # If mcworkload=get, first run mcperf with set requests to full up the
        # cache.  For each (tenant, server) pair, create a separate mcperf
        # instance on each client.
        if self.opts("mcworkload") == "get":

            tmp_assigned_cpus = 0
            hlist.mkdir(e("logs_unused"))
            for tenant in xrange(0, self.opts("mctenants")):
                for (srv_id, hserver) in enumerate(hservers.lst):
                    server_ip = socket.gethostbyname(hserver.hostname())
                    for (cli_id, hclient) in enumerate(hclients.lst):

                        # Index of tenant and client connecting to this
                        # particular server for this tenant.
                        tenant_id = "%d_%d" % (tenant, self.opts("mctenants"))
                        client_id = "%d_%d" % (cli_id, len(hclients.lst))

                        # We use the traffic class number as the seed for the
                        # random number generator so that all instances
                        # generate unique random number sequences.
                        seed = (
                            start_port +
                            (tenant * len(hservers.lst) * len(hclients.lst)) +
                            (srv_id * len(hclients.lst)) + (cli_id))

                        self.start_mcperf(hclient,
                                          server_ip,
                                          tenant_id,
                                          client_id,
                                          seed=seed,
                                          port=start_port + tenant,
                                          time=MC_PREPOPULATE_TIME,
                                          nconn=self.opts("mcnconn"),
                                          mcrate=5000,
                                          mcexp=self.opts("mcexp"),
                                          workload="set",
                                          mcsize=self.opts("mcsize"),
                                          cpus=[
                                              avail_cpus[tmp_assigned_cpus %
                                                         len(avail_cpus)]
                                          ],
                                          dir=e('logs_unused'))
                tmp_assigned_cpus += 1

            self.log(T.colored("Populating caches first", "blue"))
            progress(MC_PREPOPULATE_TIME + 5)

        # Configure rate limits
        # mcperf tenants:
        # On server, configure separate rate limit to each mctenant's client
        # On client, configure separate rate limit to each mctenant's server
        # trafgen tenants:
        # On each host, configure separate rate limits for traffic to each
        # other host, for each trafgentenant
        trafgen_pair_rate = (0 if not self.opts("trafgentenants") else
                             (self.opts("trafgen_total_rate") * 1.0 /
                              (self.opts("trafgentenants") *
                               (len(hlist.lst) - 1))))
        # Client to server traffic
        mc_pair_rate_client = (0 if not self.opts("mctenants") else
                               (self.opts("mc_total_rate_client") * 1.0 /
                                (self.opts("mctenants") * len(hservers.lst))))
        # Server to client traffic
        mc_pair_rate_server = (0 if not self.opts("mctenants") else
                               (self.opts("mc_total_rate_server") * 1.0 /
                                (self.opts("mctenants") * len(hclients.lst))))

        self.log(
            T.colored("Pair rate mc client = %s" % mc_pair_rate_client,
                      "blue"))
        self.log(
            T.colored("Pair rate mc server = %s" % mc_pair_rate_server,
                      "blue"))
        self.log(
            T.colored("Trafgen pair rate = %s" % trafgen_pair_rate, "blue"))
        if self.opts("rl") == "htb":
            hlist.mc_add_htb_qdisc(self.opts("htb_mtu"))
        elif self.opts("rl") == "qfq":
            hlist.mc_add_qfq_qdisc(self.opts("mtu"))
        elif self.opts("rl") == "eyeq":
            hlist.mc_add_htb_qdisc(self.opts("mtu"), True)

        # Qdisc classes
        # class 1 : default class
        # Separate class for each (mctenant, srv_id, cli_id) tuple
        # (start_port +
        #  (tenant * num_servers * num_clients) +
        #  (srv_id * num_clients) +
        #  (cli_id)) :  On client, this represents traffic to srv_id for tenant
        #               On server, this represents traffic to cli_id for tenant
        # Separate class for each (trafgentenant, hsrc, hdst) tuple
        # (start_port + 10000 +
        #  (tenant * num_hosts * (num_hosts - 1)) +
        #  (src_id * (num_hosts - 1)) +
        #  (dst_id)) :  Traffic from src_host to dst_host for each tenant

        for tenant in xrange(0, self.opts("mctenants")):

            for (srv_id, hserver) in enumerate(hservers.lst):
                server_ip = socket.gethostbyname(hserver.hostname())

                for (cli_id, hclient) in enumerate(hclients.lst):
                    client_ip = socket.gethostbyname(hclient.hostname())

                    srv_port = start_port + tenant
                    rate_str_client = '%.3fMbit' % mc_pair_rate_client
                    rate_str_server = '%.3fMbit' % mc_pair_rate_server
                    klass = (start_port +
                             (tenant * len(hservers.lst) * len(hclients.lst)) +
                             (srv_id * len(hclients.lst)) + (cli_id))

                    if self.opts("rl") in ["htb", "eyeq"]:
                        hclient.mc_add_htb_class(rate=rate_str_client,
                                                 ceil=rate_str_client,
                                                 klass=klass,
                                                 htb_mtu=self.opts("htb_mtu"))
                        hserver.mc_add_htb_class(rate=rate_str_server,
                                                 ceil=rate_str_server,
                                                 klass=klass,
                                                 htb_mtu=self.opts("htb_mtu"))
                    elif self.opts("rl") == "qfq":
                        hclient.mc_add_qfq_class(rate=mc_pair_rate_client,
                                                 klass=klass,
                                                 mtu=self.opts("mtu"))
                        hserver.mc_add_qfq_class(rate=mc_pair_rate_server,
                                                 klass=klass,
                                                 mtu=self.opts("mtu"))

                    if self.opts("rl") in ["htb", "eyeq", "qfq"]:
                        # Client -> Server traffic
                        hclient.mc_add_qdisc_filter(server_ip,
                                                    sport=0,
                                                    dport=srv_port,
                                                    klass=klass)
                        # Server -> Client traffic
                        hserver.mc_add_qdisc_filter(client_ip,
                                                    sport=srv_port,
                                                    dport=0,
                                                    klass=klass)

        for tenant in xrange(0, self.opts("trafgentenants")):
            trafgen_port = start_port + 1000 + tenant
            for (src_id, hsrc) in enumerate(hlist.lst):
                src_ip = socket.gethostbyname(hsrc.hostname())
                for (dst_id, hdst) in enumerate(hlist.lst):
                    if hsrc == hdst:
                        continue

                    # NOTE: Some klass ID's would be unused (when src_id ==
                    # dst_id).
                    # NOTE: Trafgen server -> client traffic (only ACKs) is
                    # allocated a rate limit of only 5Mbit.
                    dst_ip = socket.gethostbyname(hdst.hostname())
                    rate_str = '%.3fMbit' % trafgen_pair_rate
                    rate_str_acks = '5Mbit'
                    klass = (start_port + 10000 +
                             (tenant * len(hlist.lst) * len(hlist.lst)) +
                             (src_id * len(hlist.lst)) + (dst_id))

                    if self.opts("rl") in ["htb", "eyeq"]:
                        hsrc.mc_add_htb_class(rate=rate_str,
                                              ceil=rate_str,
                                              klass=klass,
                                              htb_mtu=self.opts("htb_mtu"))
                        hdst.mc_add_htb_class(rate=rate_str_acks,
                                              ceil=rate_str_acks,
                                              klass=klass,
                                              htb_mtu=self.opts("htb_mtu"))
                    elif self.opts("rl") == "qfq":
                        hsrc.mc_add_qfq_class(rate=trafgen_pair_rate,
                                              klass=klass,
                                              mtu=self.opts("mtu"))
                        hdst.mc_add_qfq_class(rate=5,
                                              klass=klass,
                                              mtu=self.opts("mtu"))

                    if self.opts("rl") in ["htb", "eyeq", "qfq"]:
                        # Trafgen client -> server traffic filter
                        hsrc.mc_add_qdisc_filter(dst_ip,
                                                 sport=0,
                                                 dport=trafgen_port,
                                                 klass=klass)
                        # Trafgen server -> client traffic filter (only affects ACKs)
                        hdst.mc_add_qdisc_filter(src_ip,
                                                 sport=trafgen_port,
                                                 dport=0,
                                                 klass=klass)

        hlist.start_bw_monitor(e('logs'))
        hlist.start_mpstat(e('logs'))
        hlist.start_perf_monitor(e('logs'), self.opts("t"))
        hlist.set_mtu(self.opts("mtu"))
        if sniffer:
            hsniffer.start_sniffer_delayed(e('logs',
                                             tmpdir=config['SNIFFER_TMPDIR']),
                                           board=0,
                                           delay=config['SNIFFER_DELAY'],
                                           duration=config['SNIFFER_DURATION'])
        sleep(1)

        # Start trafgen clients to generate background all-to-all traffic.
        # For each (tenant, destination) pair, create a separate trafgen
        # instance on the source host. This is required since trafgen currently
        # only supports a single destination per instance.
        for tenant in xrange(0, self.opts("trafgentenants")):
            tenant_id = "%d_%d" % (tenant, self.opts("trafgentenants"))
            for hsrc in hlist.lst:
                for hdst in hlist.lst:
                    if hsrc == hdst:
                        continue
                    dst_ip = socket.gethostbyname(hdst.hostname())

                    self.start_trafgen_client(
                        hsrc,
                        dst_ip,
                        tenant_id=tenant_id,
                        proto=self.opts("trafgenproto"),
                        port=start_port + 1000 + tenant,
                        mtu=self.opts("mtu"),
                        cpus=[avail_cpus[assigned_cpus % len(avail_cpus)]],
                        dir=e('logs'))
            assigned_cpus += 1

        # Start mcperf clients to generate requests. For each (tenant, server)
        # pair, create a separate mcperf instance. This is required since mcperf
        # does not have an option to send requests randomly to the available
        # memcached servers.
        tmp_assigned_cpus = 0
        for tenant in xrange(0, self.opts("mctenants")):
            for hserver in hservers.lst:
                server_ip = socket.gethostbyname(hserver.hostname())
                for (cli_id, hclient) in enumerate(hclients.lst):

                    # Index of tenant and client connecting to this particular
                    # server for this tenant.
                    tenant_id = "%d_%d" % (tenant, self.opts("mctenants"))
                    client_id = "%d_%d" % (cli_id, len(hclients.lst))

                    # We use traffic class number as the seed for the
                    # random number generator so that all instances
                    # generate unique random number sequences. But we want to
                    # use a different seed that that used for the initial "set"
                    # workload for populating caches. So we increment the
                    # respective class numbers by say 11213 to get the seed.
                    seed = (start_port + 11213 +
                            (tenant * len(hservers.lst) * len(hclients.lst)) +
                            (srv_id * len(hclients.lst)) + (cli_id))

                    self.start_mcperf(
                        hclient,
                        server_ip,
                        tenant_id,
                        client_id,
                        seed=seed,
                        port=start_port + tenant,
                        time=self.opts("t"),
                        nconn=self.opts("mcnconn"),
                        mcrate=self.opts("mcrate"),
                        mcexp=self.opts("mcexp"),
                        workload=self.opts("mcworkload"),
                        mcsize=self.opts("mcsize"),
                        cpus=[avail_cpus[tmp_assigned_cpus % len(avail_cpus)]],
                        dir=e('logs'))
            tmp_assigned_cpus += 1

        self.hservers = hservers
        self.hclients = hclients
        self.hlist = hlist
        if sniffer:
            self.hsniffer = hsniffer