def network_io_test(itype1, image1, region1, itype2, image2, region2, filesize=64, iteration=1, timeout=600):
ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.ssh/')
if itype1 in ec2.ec2_instance_types:
inst1 = ec2.EC2Inst(itype1, image1, region1, 'ubuntu', '%s/perf-bench-%s.pem'\
% (ssh_path, region1), 'perf-bench-%s' % region1)
if itype1 in gce.gce_instance_types:
inst1 = gce.GCEInst(itype1, image1, region1, os.environ['USER'], '%s/google_compute_engine' % ssh_path)
if itype2 in ec2.ec2_instance_types:
inst2 = ec2.EC2Inst(itype2, image2, region2, 'ubuntu', '%s/perf-bench-%s.pem'\
% (ssh_path, region2), 'perf-bench-%s' % region2)
if itype2 in gce.gce_instance_types:
inst2 = gce.GCEInst(itype2, image2, region2, os.environ['USER'], '%s/google_compute_engine' % ssh_path)
inst1.launch()
inst2.launch()
try:
print '[IP] waiting'
for i in range(150):
inst1.update()
inst2.update()
if inst1.remote_ip != None and inst2.remote_ip != None:
print '[IP] ok'
break
time.sleep(2)
print '[SSH] waiting'
for i in range(120):
try:
telnetlib.Telnet(inst1.remote_ip, 22, 1)
telnetlib.Telnet(inst2.remote_ip, 22, 1)
print '[SSH] ok'
break
except:
time.sleep(2)
print '[UP] %s | %s | %s' % (inst1.itype, inst1.region, inst1.remote_ip)
print '[UP] %s | %s | %s' % (inst2.itype, inst2.region, inst2.remote_ip)
util.instances_prepare([inst1, inst2], ['iperf', 'screen'])
ssh_cli = paramiko.SSHClient()
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_cli.connect(inst1.remote_ip, username=inst1.user, key_filename=inst1.ssh_key)
scp_cli = scp.SCPClient(ssh_cli.get_transport())
file_path = os.path.dirname(os.path.abspath(__file__))
scp_cli.put('%s/netcattest.py' % file_path, '/tmp/netcattest.py')
scp_cli.put('%s/scptest.py' % file_path, '/tmp/scptest.py')
scp_cli.put('%s/iperftest.py' % file_path, '/tmp/iperftest.py')
print '[START TESTS] %s %s <-----> %s %s'\
% (inst1.itype, inst1.region, inst2.itype, inst2.region)
for i in range(iteration):
print '[START ITERATION %s]' % i
print '[START] netcat'
stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/netcattest.py -i %s -u %s -k %s -s %s -t %s'
% (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout))
time.sleep(10)
for _ in range(timeout / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command('[ -f netcat.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command('cat netcat.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1':inst1.itype, 'inst2':inst2.itype})
report.update({'region1':inst1.region, 'region2':inst2.region})
report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud})
report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/netcat')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['time']
break
else:
time.sleep(5)
print '[END] netcat'
print '[START] scp'
stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/scptest.py -i %s -u %s -k %s -s %s -t %s'
% (inst2.remote_ip, inst2.user, inst2.ssh_key, filesize, timeout))
time.sleep(10)
for _ in range(timeout / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command('[ -f scp.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command('cat scp.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1':inst1.itype, 'inst2':inst2.itype})
report.update({'region1':inst1.region, 'region2':inst2.region})
report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud})
report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/scp')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['time']
break
else:
time.sleep(5)
print '[END] scp'
print '[START] iperf'
threads = [1, 4, 8]
work_time = 5
stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/iperftest.py -i %s -u %s -k %s -p %s -t %s'
% (inst2.remote_ip, inst2.user, inst2.ssh_key, ' '.join(map(str, threads)), work_time))
time.sleep(10)
for _ in range(len(threads) * (work_time + 10) / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command('[ -f iperf.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command('cat iperf.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1':inst1.itype, 'inst2':inst2.itype})
report.update({'region1':inst1.region, 'region2':inst2.region})
report.update({'cloud1':inst1.cloud, 'cloud2':inst2.cloud})
report_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../results/network-io/iperf')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open('%s/%s-%s__%s-%s' % (report_path, inst1.itype, inst1.region, inst2.itype, inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['speed']
break
else:
time.sleep(5)
print '[END] iperf'
ssh_cli.close()
except Exception:
print '[EXCEPTION] %s\n' % traceback.print_exc()
finally:
inst1.terminate()
inst2.terminate()
def disk_io_test(itype, image, region, filesize=1, mode=['randrw'], bs=[1], depth=[1], runtime=60):
ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../.ssh/')
if itype in ec2.ec2_instance_types:
inst = ec2.EC2Inst(itype, image, region, 'ec2-user', '%s/perf-bench-%s.pem'\
% (ssh_path, region), 'perf-bench-%s' % region)
if itype in gce.gce_instance_types:
inst = gce.GCEInst(itype, image, region, os.environ['USER'], '%s/google_compute_engine' % ssh_path)
report = []
utcnow = datetime.datetime.utcnow()
time_str = utcnow.strftime('%d:%m:%Y-%H:%M:%S')
inst.launch(disk_size=(filesize + 5))
try:
print '[IP] waiting'
for i in range(150):
inst.update()
if inst.remote_ip != None:
print '[IP] ok'
break
time.sleep(2)
print '[SSH] waiting'
for i in range(120):
try:
telnetlib.Telnet(inst.remote_ip, 22, 1)
print '[SSH] ok'
break
except:
time.sleep(2)
print '[UP] %s | %s | %s' % (inst.itype, inst.region, inst.remote_ip)
if inst.distr == 'centos':
util.instances_prepare([inst], ['python27', 'fio', 'screen'])
else:
util.instances_prepare([inst], ['fio', 'screen'])
ssh_cli = paramiko.SSHClient()
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_cli.connect(inst.remote_ip, username=inst.user, key_filename=inst.ssh_key)
scp_cli = scp.SCPClient(ssh_cli.get_transport())
file_path = os.path.dirname(os.path.abspath(__file__))
scp_cli.put('%s/fio_conf_generator.py' % file_path, '/tmp/fio_conf_generator.py')
scp_cli.put('%s/fiotest.py' % file_path, '/tmp/fiotest.py')
mode_str = ' '.join(mode)
bs_str = ' '.join(map(str, bs))
depth_str = ' '.join(map(str, depth))
print '[START] fio'
stdin, stdout, stderr = ssh_cli.exec_command('python2.7 /tmp/fiotest.py -f %s -s %s -m %s -b %s -d %s -t %s'
% ('/dev/sdb', filesize * 1024, mode_str, bs_str, depth_str, runtime))
for _ in range(((runtime + 30) * len(mode) * len(bs) * len(depth) + 10) / 5):
stdin, stdout, stderr = ssh_cli.exec_command('[ -s fio.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command('cat fio.report')
report = json.loads(stdout.read())
for test in report:
test.update({'instance':itype})
test.update({'region':region})
if itype in ec2.ec2_instance_types:
test.update({'cloud':'ec2'})
if itype in gce.gce_instance_types:
test.update({'cloud':'gce'})
break
else:
time.sleep(5)
print '[END] fio'
ssh_cli.close()
except Exception, e:
report.append({'error':str(e)})
print '[EXCEPTION] %s\n' % traceback.print_exc()
def network_io_test(itype1,
image1,
region1,
itype2,
image2,
region2,
filesize=64,
iteration=1,
timeout=600):
ssh_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../.ssh/')
if itype1 in ec2.ec2_instance_types:
inst1 = ec2.EC2Inst(itype1, image1, region1, 'ubuntu', '%s/perf-bench-%s.pem'\
% (ssh_path, region1), 'perf-bench-%s' % region1)
if itype1 in gce.gce_instance_types:
inst1 = gce.GCEInst(itype1, image1, region1, os.environ['USER'],
'%s/google_compute_engine' % ssh_path)
if itype2 in ec2.ec2_instance_types:
inst2 = ec2.EC2Inst(itype2, image2, region2, 'ubuntu', '%s/perf-bench-%s.pem'\
% (ssh_path, region2), 'perf-bench-%s' % region2)
if itype2 in gce.gce_instance_types:
inst2 = gce.GCEInst(itype2, image2, region2, os.environ['USER'],
'%s/google_compute_engine' % ssh_path)
inst1.launch()
inst2.launch()
try:
print '[IP] waiting'
for i in range(150):
inst1.update()
inst2.update()
if inst1.remote_ip != None and inst2.remote_ip != None:
print '[IP] ok'
break
time.sleep(2)
print '[SSH] waiting'
for i in range(120):
try:
telnetlib.Telnet(inst1.remote_ip, 22, 1)
telnetlib.Telnet(inst2.remote_ip, 22, 1)
print '[SSH] ok'
break
except:
time.sleep(2)
print '[UP] %s | %s | %s' % (inst1.itype, inst1.region,
inst1.remote_ip)
print '[UP] %s | %s | %s' % (inst2.itype, inst2.region,
inst2.remote_ip)
util.instances_prepare([inst1, inst2], ['iperf', 'screen'])
ssh_cli = paramiko.SSHClient()
ssh_cli.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_cli.connect(inst1.remote_ip,
username=inst1.user,
key_filename=inst1.ssh_key)
scp_cli = scp.SCPClient(ssh_cli.get_transport())
file_path = os.path.dirname(os.path.abspath(__file__))
scp_cli.put('%s/netcattest.py' % file_path, '/tmp/netcattest.py')
scp_cli.put('%s/scptest.py' % file_path, '/tmp/scptest.py')
scp_cli.put('%s/iperftest.py' % file_path, '/tmp/iperftest.py')
print '[START TESTS] %s %s <-----> %s %s'\
% (inst1.itype, inst1.region, inst2.itype, inst2.region)
for i in range(iteration):
print '[START ITERATION %s]' % i
print '[START] netcat'
stdin, stdout, stderr = ssh_cli.exec_command(
'python2.7 /tmp/netcattest.py -i %s -u %s -k %s -s %s -t %s' %
(inst2.remote_ip, inst2.user, inst2.ssh_key, filesize,
timeout))
time.sleep(10)
for _ in range(timeout / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command(
'[ -f netcat.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command(
'cat netcat.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1': inst1.itype, 'inst2': inst2.itype})
report.update({
'region1': inst1.region,
'region2': inst2.region
})
report.update({
'cloud1': inst1.cloud,
'cloud2': inst2.cloud
})
report_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'../results/network-io/netcat')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open(
'%s/%s-%s__%s-%s' %
(report_path, inst1.itype, inst1.region, inst2.itype,
inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['time']
break
else:
time.sleep(5)
print '[END] netcat'
print '[START] scp'
stdin, stdout, stderr = ssh_cli.exec_command(
'python2.7 /tmp/scptest.py -i %s -u %s -k %s -s %s -t %s' %
(inst2.remote_ip, inst2.user, inst2.ssh_key, filesize,
timeout))
time.sleep(10)
for _ in range(timeout / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command(
'[ -f scp.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command(
'cat scp.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1': inst1.itype, 'inst2': inst2.itype})
report.update({
'region1': inst1.region,
'region2': inst2.region
})
report.update({
'cloud1': inst1.cloud,
'cloud2': inst2.cloud
})
report_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'../results/network-io/scp')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open(
'%s/%s-%s__%s-%s' %
(report_path, inst1.itype, inst1.region, inst2.itype,
inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['time']
break
else:
time.sleep(5)
print '[END] scp'
print '[START] iperf'
threads = [1, 4, 8]
work_time = 5
stdin, stdout, stderr = ssh_cli.exec_command(
'python2.7 /tmp/iperftest.py -i %s -u %s -k %s -p %s -t %s' %
(inst2.remote_ip, inst2.user, inst2.ssh_key, ' '.join(
map(str, threads)), work_time))
time.sleep(10)
for _ in range(len(threads) * (work_time + 10) / 5 + 1):
stdin, stdout, stderr = ssh_cli.exec_command(
'[ -f iperf.report ]; echo $?')
out = stdout.read()
if out.strip() == '0':
stdin, stdout, stderr = ssh_cli.exec_command(
'cat iperf.report')
out = stdout.read()
report = json.loads(out)
report.update({'inst1': inst1.itype, 'inst2': inst2.itype})
report.update({
'region1': inst1.region,
'region2': inst2.region
})
report.update({
'cloud1': inst1.cloud,
'cloud2': inst2.cloud
})
report_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'../results/network-io/iperf')
if not os.path.exists(report_path):
cmd = 'mkdir -p %s' % report_path
subps.call(cmd.split())
#os.mkdir(report_path)
with open(
'%s/%s-%s__%s-%s' %
(report_path, inst1.itype, inst1.region, inst2.itype,
inst2.region), 'a+') as f:
f.write(json.dumps(report, indent=4, sort_keys=True))
f.write('\n')
print report['speed']
break
else:
time.sleep(5)
print '[END] iperf'
ssh_cli.close()
except Exception:
print '[EXCEPTION] %s\n' % traceback.print_exc()
finally:
inst1.terminate()
inst2.terminate()
