def prepare_unixbench_test(options, ovc, cpu_cores, machine_id, publicip,
publicport):
print("Preparing unixbench test on machine {}".format(machine_id))
machine = safe_get_vm(ovc, concurrency, machine_id)
machines[machine_id] = machine['name']
account = machine['accounts'][0]
wait_until_remote_is_listening(publicip, int(publicport), True, machine_id)
# templ = 'sshpass -p{} scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null '
# templ += '-P {} {}/2_Unixbench2_test/2_machine_script.py {}@{}:'
# cmd = templ.format(account['password'], publicport, options.testsuite, account['login'], publicip)
# run_cmd_via_gevent(cmd)
return machine_id, publicip, publicport, account, cpu_cores
def main(options):
from JumpScale import j
# Check dependencies
if not os.path.exists(options.results_dir):
print(
"Not all dependencies are met. Make sure the result directory exists."
)
return
if not check_package('sshpass') or not check_package(
'python3-prettytable'):
return
# Prepare test run
hostname = run_cmd_via_gevent('hostname').replace("\n", "")
test_num = len(os.listdir('{}'.format(options.results_dir))) + 1
test_dir = "/" + datetime.datetime.today().strftime('%Y-%m-%d')
test_dir += "_" + hostname + "_testresults_{}".format(test_num)
results_dir = options.results_dir + test_dir
run_cmd_via_gevent('mkdir -p {}'.format(results_dir))
# list virtual and deployed cloudspaces
vms = []
vms_index = set()
ovc = j.clients.openvcloud.get(options.environment, options.username,
options.password)
cloudspaces_per_user = ovc.api.cloudapi.cloudspaces.list()
for cs in cloudspaces_per_user:
portforwards = ovc.api.cloudapi.portforwarding.list(
cloudspaceId=cs['id'])
for pi in portforwards:
if 'machineId' in pi and not pi['machineId'] in vms_index:
vms.append([pi['machineId'], pi['publicIp'], pi['publicPort']])
vms_index.add(pi['machineId'])
if len(vms) < options.required_vms:
print("Not enough vms available to run this test.")
return
vms = vms[:options.required_vms]
# getting bootdisk size, cpu and memory used during vms creatian (for any vm)
machine = safe_get_vm(ovc, concurrency, pi['machineId'])
bootdisk = machine['disks'][0]['sizeMax']
size_id = machine['sizeid']
sizes = ovc.api.cloudapi.sizes.list(cloudspaceId=cs['id'])
memory = next((i for i in sizes if i['id'] == size_id), False)['memory']
cpu = next((i for i in sizes if i['id'] == size_id), False)['vcpus']
# prepare unixbench tests
prepare_jobs = [
gevent.spawn(prepare_unixbench_test, options, ovc, cpu, *vm)
for vm in vms
]
gevent.joinall(prepare_jobs)
# run unixbench tests
run_jobs = [
gevent.spawn(unixbench_test, options, c, *job.value) for job, c in zip(
*[prepare_jobs, range(len(prepare_jobs))]) if job.value is not None
]
gevent.joinall(run_jobs)
raw_results = [job.value for job in run_jobs if job.value]
raw_results.sort(key=lambda x: x[1])
results = list()
index = 0
for s in raw_results:
index += 1
results.append([
index, '{} (id={})'.format(machines.get(s[0], s[0]), s[0]), cpu,
memory, bootdisk, s[1]
])
titles = [
'Index', 'VM', 'CPU\'s', 'Memory(MB)', 'HDD(GB)',
'Avg. Unixbench Score'
]
collect_results(titles, results, results_dir, 'average-results')
titles = ['VM', 'Timestamp (epoch)', 'Score']
results = list()
for result in raw_results:
machine_id, avg_score, scores = result
for timestamp, score in scores:
results.append([
'{} (id={})'.format(machines.get(machine_id, machine_id),
machine_id), timestamp, score
])
collect_results(titles, results, results_dir, 'all-results')
# report results
with open(os.path.join(results_dir, 'parameters.md'), 'w') as params:
params.write("# Parameters\n\n")
for key, value in vars(options).items():
params.write("- **{}**: {}\n".format(key, value))
# pushing results to env_repo
location = options.environment.split('.')[0]
push_results_to_repo(results_dir, location)
def deploy_vm(self,
name,
cloudspace_id,
template_details,
datadisk=None,
iops=None,
size_id=None):
# Listing sizes
if size_id is None:
raise ValueError("No matching size for vm found.")
if options.iops < 0:
raise ValueError("Maximum iops can't be a negative value")
# Create vm
with concurrency:
datadisks = [int(datadisk)] if datadisk else []
if not isinstance(template_details, tuple):
print("Creating {}".format(name))
vm_id = self.ovc.api.cloudapi.machines.create(
cloudspaceId=cloudspace_id,
name=name,
description=name,
sizeId=size_id,
imageId=template_details,
disksize=options.bootdisk,
datadisks=datadisks)
limitIOdone = False
else:
print("Cloning {}".format(name))
machine_id, snapshot_timestamp = template_details
vm_id = self.ovc.api.cloudapi.machines.clone(
machineId=machine_id,
name=name,
cloudspaceId=cloudspace_id,
snapshotTimestamp=snapshot_timestamp)
print("Adding data disk to {}".format(name))
disk_id = self.ovc.api.cloudapi.machines.addDisk(
machineId=vm_id,
diskName="data",
description="workhorse",
size=int(datadisk),
type="D")
if iops is not None:
print(
"Set limit of iops to {} on disk {}({}) for machine {}"
.format(options.iops, "data", disk_id, name))
self.ovc.api.cloudapi.disks.limitIO(diskId=disk_id,
iops=iops)
limitIOdone = True
# limit the IOPS on all the disks of the vm
machine = safe_get_vm(self.ovc, concurrency, vm_id)
if iops is not None and not limitIOdone:
for disk in machine['disks']:
if disk['type'] != 'D':
continue
with concurrency:
print(
"Set limit of iops to {} on disk {}({}) for machine {}"
.format(options.iops, disk['name'], disk['id'], name))
self.ovc.api.cloudapi.disks.limitIO(diskId=disk['id'],
iops=iops)
# Wait until vm has ip address
start = time.time()
while True:
gevent.sleep(5)
machine = safe_get_vm(self.ovc, concurrency, vm_id)
ip = machine['interfaces'][0]['ipAddress']
if ip != 'Undefined':
break
now = time.time()
if now > start + 600:
raise RuntimeError(
"Machine {} did not get an ip within 600 seconds".format(
vm_id))
print("Waiting {} seconds for an IP for VM {}".format(
int(now - start), name))
# Configure portforward to ssh port of vm
print("Configuring portforward for machine {}".format(name))
cloudspace = self.ovc.api.cloudapi.cloudspaces.get(
cloudspaceId=cloudspace_id)
with get_publicport_semaphore(cloudspace_id):
public_ports = [
int(pf['publicPort'])
for pf in self.ovc.api.cloudapi.portforwarding.list(
cloudspaceId=cloudspace_id)
]
public_ports.append(19999)
public_port = max(public_ports) + 1
with concurrency:
self.ovc.api.cloudapi.portforwarding.create(
cloudspaceId=cloudspace_id,
publicIp=cloudspace['publicipaddress'],
publicPort=public_port,
machineId=vm_id,
localPort=22,
protocol='tcp')
machine['public_port'] = public_port
print("Machine {} deployed succesfully.".format(name))
_stats['deployed_vms'] += 1
return machine