def network_io_test(itype1, image1, region1, itype2, image2, region2, filesize=64, iteration=1, timeout=600): ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.ssh/') if itype1 in ec2.ec2_instance_types: inst1 = ec2.EC2Inst(itype1, image1, region1, 'ubuntu', '%s/perf-bench-%s.pem'\ % (ssh_path, region1), 'perf-bench-%s' % region1) if itype1 in gce.gce_instance_types: inst1 = gce.GCEInst(itype1, image1, region1, os.environ['USER'], '%s/google_compute_engine' % ssh_path) if itype2 in ec2.ec2_instance_types: inst2 = ec2.EC2Inst(itype2, image2, region2, 'ubuntu', '%s/perf-bench-%s.pem'\ % (ssh_path, region2), 'perf-bench-%s' % region2) if itype2 in gce.gce_instance_types: inst2 = gce.GCEInst(itype2, image2, region2, os.environ['USER'], '%s/google_compute_engine' % ssh_path) inst1.launch() inst2.launch() try: print '[IP] waiting' for i in range(150): inst1.update() inst2.update() if inst1.remote_ip != None and inst2.remote_ip != None: print '[IP] ok' break time.sleep(2) print '[SSH] waiting' for i in range(120): try: telnetlib.Telnet(inst1.remote_ip, 22, 1) telnetlib.Telnet(inst2.remote_ip, 22, 1) print '[SSH] ok' break except: time.sleep(2) print '[UP] %s | %s | %s' % (inst1.itype, inst1.region, inst1.remote_ip) print '[UP] %s | %s | %s' % (inst2.itype, inst2.region, inst2.remote_ip) util.instances_prepare([inst1, inst2], ['iperf', 'screen']) ssh_cli = paramiko.SSHClient() ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_cli.connect(inst1.remote_ip, username=inst1.user, key_filename=inst1.ssh_key) scp_cli = scp.SCPClient(ssh_cli.get_transport()) file_path = os.path.dirname(os.path.abspath(__file__)) scp_cli.put('%s/netcattest.py' % file_path, '/tmp/netcattest.py') scp_cli.put('%s/scptest.py' % file_path, '/tmp/scptest.py') scp_cli.put('%s/iperftest.py' % file_path, '/tmp/iperftest.py') print '[START TESTS] %s %s <-----> %s %s'\ % (inst1.itype, inst1.region, inst2.itype, inst2.region) for i in range(iteration): print '[START ITERATION %s]' % i print '[START] netcat' stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/netcattest.py -i %s -u %s -k %s -s %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout)) time.sleep(10) for _ in range(timeout / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command('[ -f netcat.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command('cat netcat.report') out = stdout.read() report = json.loads(out) report.update({'inst1':inst1.itype, 'inst2':inst2.itype}) report.update({'region1':inst1.region, 'region2':inst2.region}) report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud}) report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/netcat') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['time'] break else: time.sleep(5) print '[END] netcat' print '[START] scp' stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/scptest.py -i %s -u %s -k %s -s %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout)) time.sleep(10) for _ in range(timeout / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command('[ -f scp.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command('cat scp.report') out = stdout.read() report = json.loads(out) report.update({'inst1':inst1.itype, 'inst2':inst2.itype}) report.update({'region1':inst1.region, 'region2':inst2.region}) report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud}) report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/scp') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['time'] break else: time.sleep(5) print '[END] scp' print '[START] iperf' threads = [1, 4, 8] work_time = 5 stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/iperftest.py -i %s -u %s -k %s -p %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, ' '.join(map(str, threads)), work_time)) time.sleep(10) for _ in range(len(threads) * (work_time + 10) / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command('[ -f iperf.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command('cat iperf.report') out = stdout.read() report = json.loads(out) report.update({'inst1':inst1.itype, 'inst2':inst2.itype}) report.update({'region1':inst1.region, 'region2':inst2.region}) report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud}) report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/iperf') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['speed'] break else: time.sleep(5) print '[END] iperf' ssh_cli.close() except Exception: print '[EXCEPTION] %s\n' % traceback.print_exc() finally: inst1.terminate() inst2.terminate()
def disk_io_test(itype, image, region, filesize=1, mode=['randrw'], bs=[1], depth=[1], runtime=60): ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.ssh/') if itype in ec2.ec2_instance_types: inst = ec2.EC2Inst(itype, image, region, 'ec2-user', '%s/perf-bench-%s.pem'\ % (ssh_path, region), 'perf-bench-%s' % region) if itype in gce.gce_instance_types: inst = gce.GCEInst(itype, image, region, os.environ['USER'], '%s/google_compute_engine' % ssh_path) report = [] utcnow = datetime.datetime.utcnow() time_str = utcnow.strftime('%d:%m:%Y-%H:%M:%S') inst.launch(disk_size=(filesize + 5)) try: print '[IP] waiting' for i in range(150): inst.update() if inst.remote_ip != None: print '[IP] ok' break time.sleep(2) print '[SSH] waiting' for i in range(120): try: telnetlib.Telnet(inst.remote_ip, 22, 1) print '[SSH] ok' break except: time.sleep(2) print '[UP] %s | %s | %s' % (inst.itype, inst.region, inst.remote_ip) if inst.distr == 'centos': util.instances_prepare([inst], ['python27', 'fio', 'screen']) else: util.instances_prepare([inst], ['fio', 'screen']) ssh_cli = paramiko.SSHClient() ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_cli.connect(inst.remote_ip, username=inst.user, key_filename=inst.ssh_key) scp_cli = scp.SCPClient(ssh_cli.get_transport()) file_path = os.path.dirname(os.path.abspath(__file__)) scp_cli.put('%s/fio_conf_generator.py' % file_path, '/tmp/fio_conf_generator.py') scp_cli.put('%s/fiotest.py' % file_path, '/tmp/fiotest.py') mode_str = ' '.join(mode) bs_str = ' '.join(map(str, bs)) depth_str = ' '.join(map(str, depth)) print '[START] fio' stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/fiotest.py -f %s -s %s -m %s -b %s -d %s -t %s' % ('/dev/sdb', filesize * 1024, mode_str, bs_str, depth_str, runtime)) for _ in range(((runtime + 30) * len(mode) * len(bs) * len(depth) + 10) / 5): stdin, stdout, stderr = ssh_cli.exec_command('[ -s fio.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command('cat fio.report') report = json.loads(stdout.read()) for test in report: test.update({'instance':itype}) test.update({'region':region}) if itype in ec2.ec2_instance_types: test.update({'cloud':'ec2'}) if itype in gce.gce_instance_types: test.update({'cloud':'gce'}) break else: time.sleep(5) print '[END] fio' ssh_cli.close() except Exception, e: report.append({'error':str(e)}) print '[EXCEPTION] %s\n' % traceback.print_exc()
def network_io_test(itype1, image1, region1, itype2, image2, region2, filesize=64, iteration=1, timeout=600): ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.ssh/') if itype1 in ec2.ec2_instance_types: inst1 = ec2.EC2Inst(itype1, image1, region1, 'ubuntu', '%s/perf-bench-%s.pem'\ % (ssh_path, region1), 'perf-bench-%s' % region1) if itype1 in gce.gce_instance_types: inst1 = gce.GCEInst(itype1, image1, region1, os.environ['USER'], '%s/google_compute_engine' % ssh_path) if itype2 in ec2.ec2_instance_types: inst2 = ec2.EC2Inst(itype2, image2, region2, 'ubuntu', '%s/perf-bench-%s.pem'\ % (ssh_path, region2), 'perf-bench-%s' % region2) if itype2 in gce.gce_instance_types: inst2 = gce.GCEInst(itype2, image2, region2, os.environ['USER'], '%s/google_compute_engine' % ssh_path) inst1.launch() inst2.launch() try: print '[IP] waiting' for i in range(150): inst1.update() inst2.update() if inst1.remote_ip != None and inst2.remote_ip != None: print '[IP] ok' break time.sleep(2) print '[SSH] waiting' for i in range(120): try: telnetlib.Telnet(inst1.remote_ip, 22, 1) telnetlib.Telnet(inst2.remote_ip, 22, 1) print '[SSH] ok' break except: time.sleep(2) print '[UP] %s | %s | %s' % (inst1.itype, inst1.region, inst1.remote_ip) print '[UP] %s | %s | %s' % (inst2.itype, inst2.region, inst2.remote_ip) util.instances_prepare([inst1, inst2], ['iperf', 'screen']) ssh_cli = paramiko.SSHClient() ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_cli.connect(inst1.remote_ip, username=inst1.user, key_filename=inst1.ssh_key) scp_cli = scp.SCPClient(ssh_cli.get_transport()) file_path = os.path.dirname(os.path.abspath(__file__)) scp_cli.put('%s/netcattest.py' % file_path, '/tmp/netcattest.py') scp_cli.put('%s/scptest.py' % file_path, '/tmp/scptest.py') scp_cli.put('%s/iperftest.py' % file_path, '/tmp/iperftest.py') print '[START TESTS] %s %s <-----> %s %s'\ % (inst1.itype, inst1.region, inst2.itype, inst2.region) for i in range(iteration): print '[START ITERATION %s]' % i print '[START] netcat' stdin, stdout, stderr = ssh_cli.exec_command( 'python2.7 /tmp/netcattest.py -i %s -u %s -k %s -s %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout)) time.sleep(10) for _ in range(timeout / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command( '[ -f netcat.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command( 'cat netcat.report') out = stdout.read() report = json.loads(out) report.update({'inst1': inst1.itype, 'inst2': inst2.itype}) report.update({ 'region1': inst1.region, 'region2': inst2.region }) report.update({ 'cloud1': inst1.cloud, 'cloud2': inst2.cloud }) report_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '../results/network-io/netcat') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open( '%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['time'] break else: time.sleep(5) print '[END] netcat' print '[START] scp' stdin, stdout, stderr = ssh_cli.exec_command( 'python2.7 /tmp/scptest.py -i %s -u %s -k %s -s %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout)) time.sleep(10) for _ in range(timeout / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command( '[ -f scp.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command( 'cat scp.report') out = stdout.read() report = json.loads(out) report.update({'inst1': inst1.itype, 'inst2': inst2.itype}) report.update({ 'region1': inst1.region, 'region2': inst2.region }) report.update({ 'cloud1': inst1.cloud, 'cloud2': inst2.cloud }) report_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '../results/network-io/scp') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open( '%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['time'] break else: time.sleep(5) print '[END] scp' print '[START] iperf' threads = [1, 4, 8] work_time = 5 stdin, stdout, stderr = ssh_cli.exec_command( 'python2.7 /tmp/iperftest.py -i %s -u %s -k %s -p %s -t %s' % (inst2.remote_ip, inst2.user, inst2.ssh_key, ' '.join( map(str, threads)), work_time)) time.sleep(10) for _ in range(len(threads) * (work_time + 10) / 5 + 1): stdin, stdout, stderr = ssh_cli.exec_command( '[ -f iperf.report ]; echo $?') out = stdout.read() if out.strip() == '0': stdin, stdout, stderr = ssh_cli.exec_command( 'cat iperf.report') out = stdout.read() report = json.loads(out) report.update({'inst1': inst1.itype, 'inst2': inst2.itype}) report.update({ 'region1': inst1.region, 'region2': inst2.region }) report.update({ 'cloud1': inst1.cloud, 'cloud2': inst2.cloud }) report_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), '../results/network-io/iperf') if not os.path.exists(report_path): cmd = 'mkdir -p %s' % report_path subps.call(cmd.split()) #os.mkdir(report_path) with open( '%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f: f.write(json.dumps(report, indent=4, sort_keys=True)) f.write('\n') print report['speed'] break else: time.sleep(5) print '[END] iperf' ssh_cli.close() except Exception: print '[EXCEPTION] %s\n' % traceback.print_exc() finally: inst1.terminate() inst2.terminate()