Example #1
0
def last_run(args):
  container_folder = util.container_folder(args.process, args.container)
  runs = util.shell_output('ls {0:s}'.format(container_folder)).strip().split('\n')
  runs.sort(reverse=True)

  i = 0
  for run in runs:
    readme_file = '{0:s}/{1:s}/README'.format(container_folder, run)
    if not os.path.isfile(readme_file):
      continue

    output = util.shell_output('cat {0:s}'.format(readme_file))
    if not correct_num_clients(output, args.num_clients):
      continue

    if not correct_benchmark(output, args.test):
      continue
    
    print(readme_file)    
    if args.output:
      print(output)
    i += 1
    if i == args.instances:
      return
  sys.exit(1)
Example #2
0
def check_processor(args, name):
  if args.container == "docker":
    print("here")
    command = "sudo docker inspect -f '{{{{.HostConfig.CpusetCpus}}}}' {0:s}".format(name)
    print(command)
    output = util.shell_output(command).strip()
    print(output)
  elif args.container == "linux":
    output = util.shell_output("lxc-cgroup -n {0:s} cpuset.cpus".format(name)).strip()
  else:
    return
  print(output)
  if output != str(util.processor(0)):
    raise Exception("Error. Container is not bound to processor {0:d}".format(util.processor(0)))
Example #3
0
def setup_xcontainer(args):
  if args.process == "nginx":
    name = NGINX_CONTAINER_NAME
    container_port = NGINX_CONTAINER_PORT
    machine_port = NGINX_MACHINE_PORT
    create_docker_nginx_container(args, XCONTAINER_INSPECT_FILTER, True)
  elif args.process == "memcached":
    name = MEMCACHED_CONTAINER_NAME
    container_port = MEMCACHED_CONTAINER_PORT
    machine_port = MEMCACHED_MACHINE_PORT
    setup_docker_memcached_container(args, XCONTAINER_INSPECT_FILTER, True)
  else:
    raise Exception("setup_xcontainer: not implemented")

  docker_id = util.shell_output('docker inspect --format="{{{{.Id}}}}" {0:s}'.format(name)).strip()
  bridge_ip = get_ip_address('xenbr0')
  xcontainer_ip = generate_xcontainer_ip(bridge_ip)
  util.shell_call('docker stop {0:s}'.format(name))
  machine_ip = get_ip_address('em1')
  util.shell_call('tmux new -s xcontainer -d')
  tmux_command('xcontainer', 'cd /root/experiments/native/compute06/docker')
  tmux_command('xcontainer', 'python run.py --id {0:s} --ip {1:s} --hvm --name {2:s} --cpu={3:d}'.format(docker_id, xcontainer_ip, name, args.cores))
  container_sleep(5)
  setup_port_forwarding(machine_ip, machine_port, xcontainer_ip, container_port, bridge_ip)
  print('Setup {0:s} X-Container on {1:s}:{2:d}'.format(args.process, machine_ip, machine_port))
  util.shell_call('xl vcpu-pin memcached_container 0 {0:d}'.format(util.processor(0)))
  util.shell_call('python /root/x-container/irq-balance.py')
  util.shell_call('python /root/x-container/cpu-balance.py')
Example #4
0
 def ip(self):
     try:
         output = util.shell_output('lxc-info -n {:s} -iH'.format(
             self.name)).decode('utf-8').strip()
         if output == "":
             return None
         return output
     except subprocess.CalledProcessError as e:
         return None
Example #5
0
def check_git():
  util.shell_call('git remote update')
  output = util.shell_output("git status --untracked-files=no").strip().split("\n")
  # Clean state should look like
  # On branch master
  # Your branch is up-to-date with 'origin/master'
  # nothing to commit (use -u to show untracked files)
  if len(output) > 3:
    raise Exception("Commit your code before you run this script!")
Example #6
0
def get_linux_container_ip(name):
  try:
    output = util.shell_output('lxc-info -n {:s} -iH'.format(name))
    output = output.decode('utf-8').strip()
    if output == "":
      return None
    return output
  except subprocess.CalledProcessError as e:
    return None
Example #7
0
def docker_ip(name, docker_filter):
  print(docker_filter, name)
  try:
    output = util.shell_output("docker inspect -f '{0:s}' {1:s}".format(docker_filter, name))
    output = output.strip()
    if output == "":
      return None
    return output
  except subprocess.CalledProcessError as e:
    return None
Example #8
0
def docker_port(name, regex):
  try:
    output = util.shell_output('docker port {0:s}'.format(name))
    results = re.findall(regex, output)
    if len(results) == 0:
      return None
    else:
      return list(results[0])
  except subprocess.CalledProcessError as e:
    return None
Example #9
0
 def ip(self):
     try:
         output = util.shell_output(
             "docker inspect -f '{0:s}' {1:s}".format(
                 DOCKER_INSPECT_FILTER, self.name))
         output = output.strip()
         if output == "":
             return None
         return output
     except subprocess.CalledProcessError as e:
         return None
Example #10
0
def check_last_run(args):
  folder = util.container_folder(args.process, args.container)
  output = util.shell_output("ls {0:s}".format(folder)).strip().split("\n")
  if len(output) == 0 or (len(output) == 1 and output[0] == ''):
    return

  output.sort(reverse=True)
  for date in output:
    filename = "{0:s}/{1:s}/README".format(folder, date)
    if not os.path.isfile(filename):
      continue
    lines = open(filename).read()
    if ('BENCHMARK TEST: {0:s}\n'.format(args.test) in lines) or ('BENCHMARK TEST:' not in lines and args.test == 'bare'):
      if "NOTE: " not in lines:
        raise Exception("Need to add a note to {0:s} to explain why this needs to be rerun".format(filename))
      return
Example #11
0
  def wait_for_hosts(self):
    l = len(self.hosts)
    if self.switch == UserSwitch:
      l += len(self.switches)

    while True and not settings.dryrun:
      print 'Waiting for hosts/switches to boot..', 
      sys.stdout.flush()
      sleep(5)
      sys.stdout.flush()
      n = int(shell_output('vzlist -a | egrep "running|mounted" | wc -l').strip())
      print '%d/%d' % (n, l)
      if n > l:
        print 'There seem to be extra hosts booted up; please check.'
        print 'But I am continuing with other things...'
        break
      if n == l:
        break
Example #12
0
def create_readme(args, folder):
  last_commit = util.shell_output('git log --oneline -n 1')
  num_connections = get_num_connections(args)
  rates = get_rates(args)

  f = open('{0:s}/README'.format(folder), 'w+')
  f.write('LAST COMMIT: {0:s}\n'.format(last_commit))
  f.write('CONTAINER: {0:s}\n'.format(args.container))
  f.write('PROCESS: {0:s}\n'.format(args.process))
  f.write('BENCHMARK ADDRESS: {0:s}\n'.format(args.benchmark_address))
  f.write('NUM CLIENTS: {0:d}\n'.format(args.cores))
  f.write('DURATION: {0:d}\n'.format(args.duration))
  f.write('CONNECTIONS: {0:d}\n'.format(args.connections))
  f.write('NUM CONNECTIONS PER CLIENT: {0:d}\n'.format(num_connections))
  f.write('RATES: {0:s}\n'.format(str(rates)))
  f.write('THREADS: {0:d}\n'.format(args.threads))
  f.write('DATE: {0:s}\n'.format(args.date))
  f.write("BOUND TO PROCESSOR: {0:d}\n".format(util.processor(0)))
  f.write('BENCHMARK TEST: {0:s}\n'.format(args.test))
  f.write('NUM INTERFERENCE: {0:d}\n'.format(NUM_INTERFERENCE))
  if args.process == 'memcached':
    f.write('MEMCACHED SIZE(-m): {0:d}M\n'.format(MEMCACHED_SIZE))
    f.write('MEMCACHED THREADS(-t): {0:d}\n'.format(MEMCACHED_THREADS))
  f.close()
Example #13
0
 def id(self):
     docker_id = util.shell_output(
         'docker inspect --format="{{{{.Id}}}}" {0:s}'.format(
             self.name)).strip()
     return docker_id
Example #14
0
def get_ip_address(name):
  return util.shell_output("/sbin/ifconfig {0:s} | grep 'inet addr:' | cut -d: -f2 | awk '{{ print $1 }}'".format(name)).strip()
Example #15
0
def get_date():
  return util.shell_output('date +%F-%H-%M-%S').strip()
Example #16
0
  def output(self):
    ret = ''
    bandwidth = [ ','.join( ['Time'] + [('b%d' % i) for i in xrange(1,10+1)] )]
    tcpwindow = [ ','.join( ['Time'] + [('W%d' % i) for i in xrange(1,10+1)] )]
    bs = {}
    tcp= {}
    losses = {}

    for t in xrange(0, self.t):
      bs[t]=["%d"%t]
      tcp[t]=["%d"%t]

    for h in self.hosts[1:]:
      # time series data from iperf, tcpstats.py
      lines = h.open('iperf_output/%s-%s' %(h.name, self.hosts[0].name)).readlines()
      t=0
      for l in lines:
        val = int(l.split(',')[-1])*1.0/(2**20)
        bs[t].append("%.2f" % val)
        t += 1
        if t == self.t:
          break
      
      lines = map(lambda x: x.strip(), h.open('tcpstats.csv').readlines())[0:t]
      t=0
      for l in lines[1:]:
        val = int(l.split(',')[1])
        tcp[t].append("%d" % val)
        t += 1
        if t == self.t:
          break

      # get loss packets
      losses[h.id]={}
      lines = map(lambda x: x.strip().split(' '), 
        shell_output('vzctl exec %d cat /proc/net/netstat' % h.id).split('\n')[0:2])
        #h.open('proc/net/netstat').readlines()[0:2])
        # BUG!! theres a discrepancy between the values
        # read from /proc/net/netstat from inside the container
        # and /mountpoint/proc/net/netstat! 
        # Damn :p

      for k,v in zip(lines[0], lines[1]):
        if k in ['TCPLoss', 'TCPTimeouts']:
          losses[h.id][k]=v
      losses[h.id]['debug']=zip(lines[0],lines[1])
      
    # put values together
    for t in xrange(0,self.t):
      bandwidth.append(','.join(bs[t]))
      tcpwindow.append(','.join(tcp[t]))

    ret += html.csv(bandwidth, 
      "Bandwidth per host", 
      ["valueRange:[0,15]"])

    ret += html.csv(tcpwindow, 
      "TCP Window size hosts",
      ["rollPeriod:1", "showRoller:true"])

    # this is related to the netstat problem
    ret += html.join([html.tag("h3", "TCP stats"), html.table(losses)])
    return ret
Example #17
0
def get_known_packages():
  output = util.shell_output('dpkg --get-selections').decode('utf-8')
  lines = output.split('\n')
  packages = list(map(lambda x: x.split('\t')[0], lines))
  packages = list(filter(lambda x: len(x) > 0, packages))
  return packages