def assemble_fio_test_results(results_dir, account, publicport, cloudspace_publicip, machine_id): print('Collecting results from machine: {}'.format(machine_id)) templ = 'sshpass -p{} scp -r -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ' templ += '-P {} {}@{}:machine{}_iter{}_{}_results {}/' cmd = templ.format(account['password'], publicport, account['login'], cloudspace_publicip, machine_id, 1, options.write_type, results_dir) run_cmd_via_gevent(cmd)
def unixbench_test(options, count, machine_id, publicip, publicport, account, cpu_cores): gevent.sleep(options.time_interval * count) print( 'unixbench testing has been started on machine: {}'.format(machine_id)) templ = 'sshpass -p "{}" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {} {}@{} ' templ += ' "cd /home/cloudscalers/UnixBench; echo {} | sudo -S ./Run -c {} -i 1"' cmd = templ.format(account['password'], publicport, account['login'], publicip, account['password'], cpu_cores) results = list() start = time.time() while start + options.test_runtime > time.time(): output = run_cmd_via_gevent(cmd) match = None for line in output.splitlines(): m = matcher.match(line) if m: match = m break if match: result = float(match.group(1)) results.append((time.time(), result)) print("Machine {} reports score of {}".format(machine_id, result)) else: logger.error( "Unixbench did not return result:\n\n{}".format(output)) return machine_id, sum((r[1] for r in results)) / len(results), results
def install_req(self, machine, cloudspace): account = machine['accounts'][0] # Wait until vm accepts connections wait_until_remote_is_listening(cloudspace['publicipaddress'], machine['public_port']) # Copy install_deps.sh to vm templ = 'sshpass -p "{0}" scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' templ += ' -P {1} install_deps.sh {2}@{3}:/home/{2}' cmd = templ.format(account['password'], machine['public_port'], account['login'], cloudspace['externalnetworkip']) run_cmd_via_gevent(cmd) # Run bash script on vm templ = 'sshpass -p "{0}" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {1} {2}@{3} ' templ += '\'echo "{0}" | sudo -S bash /home/{2}/install_deps.sh\'' cmd = templ.format(account['password'], machine['public_port'], account['login'], cloudspace['externalnetworkip']) run_cmd_via_gevent(cmd)
def fio_test(options, machine_id, publicip, publicport, account): machines_running.add(machine_id) # only one data disk for this test disks_num = 1 templ = 'sshpass -p "{}" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {} {}@{} ' templ += ' python Machine_script.py {} {} {} {} {} {} {} {} {} {} {} {} {} {}' cmd = templ.format(account['password'], publicport, account['login'], publicip, options.testrun_time, machine_id, account['password'], 1, disks_num, options.data_size, options.write_type, options.block_size, options.iodepth, options.direct_io, options.rwmixwrite, options.rate_iops, options.numjobs, options.type) print('FIO testing has been started on machine: {}'.format(machine_id)) run_cmd_via_gevent(cmd) machines_complete.add(machine_id) running = machines_running.difference(machines_complete) complete = (len(machines_running) - len(running)) / len(machines_running) * 100.0 print('Testing completed for {:.2f}%'.format(complete)) if complete >= 90.0: print('Waiting for machines {} to complete their test ...'.format( ' '.join(str(x) for x in running))) return account, publicport, publicip, machine_id
def pgbench(options, machine_id, publicip, publicport, account): machines_running.add(machine_id) # only one data disk for this test templ = 'sshpass -p "{}" ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {} {}@{} ' templ += ' bash run_pgbench.sh {} {} {} {} {} {}' cmd = templ.format(account['password'], publicport, account['login'], publicip, account['password'], 'vdb', options.scalefactor, options.testrun_time, options.threadcount, options.clientcount) print('Postgress benchmarking testing has been started on machine: {}'.format(machine_id)) iops = int(run_cmd_via_gevent(cmd).splitlines()[-1]) machines_complete.add(machine_id) running = machines_running.difference(machines_complete) complete = (len(machines_running) - len(running)) / len(machines_running) * 100.0 print('Machine {} reports {} iops. Testing completed for {:.2f}%'.format(machine_id, iops, complete)) return machine_id, iops
def main(options): from JumpScale import j # Check dependencies if not os.path.exists(options.results_dir): print( "Not all dependencies are met. Make sure the result directory exists." ) return if not check_package('sshpass') or not check_package( 'python3-prettytable'): return # Prepare test run hostname = run_cmd_via_gevent('hostname').replace("\n", "") test_num = len(os.listdir('{}'.format(options.results_dir))) + 1 test_dir = "/" + datetime.datetime.today().strftime('%Y-%m-%d') test_dir += "_" + hostname + "_testresults_{}".format(test_num) results_dir = options.results_dir + test_dir run_cmd_via_gevent('mkdir -p {}'.format(results_dir)) # list virtual and deployed cloudspaces vms = [] vms_index = set() ovc = j.clients.openvcloud.get(options.environment, options.username, options.password) cloudspaces_per_user = ovc.api.cloudapi.cloudspaces.list() for cs in cloudspaces_per_user: portforwards = ovc.api.cloudapi.portforwarding.list( cloudspaceId=cs['id']) for pi in portforwards: if 'machineId' in pi and not pi['machineId'] in vms_index: vms.append([pi['machineId'], pi['publicIp'], pi['publicPort']]) vms_index.add(pi['machineId']) if len(vms) < options.required_vms: print("Not enough vms available to run this test.") return vms = vms[:options.required_vms] # getting bootdisk size, cpu and memory used during vms creatian (for any vm) machine = safe_get_vm(ovc, concurrency, pi['machineId']) bootdisk = machine['disks'][0]['sizeMax'] size_id = machine['sizeid'] sizes = ovc.api.cloudapi.sizes.list(cloudspaceId=cs['id']) memory = next((i for i in sizes if i['id'] == size_id), False)['memory'] cpu = next((i for i in sizes if i['id'] == size_id), False)['vcpus'] # prepare unixbench tests prepare_jobs = [ gevent.spawn(prepare_unixbench_test, options, ovc, cpu, *vm) for vm in vms ] gevent.joinall(prepare_jobs) # run unixbench tests run_jobs = [ gevent.spawn(unixbench_test, options, c, *job.value) for job, c in zip( *[prepare_jobs, range(len(prepare_jobs))]) if job.value is not None ] gevent.joinall(run_jobs) raw_results = [job.value for job in run_jobs if job.value] raw_results.sort(key=lambda x: x[1]) results = list() index = 0 for s in raw_results: index += 1 results.append([ index, '{} (id={})'.format(machines.get(s[0], s[0]), s[0]), cpu, memory, bootdisk, s[1] ]) titles = [ 'Index', 'VM', 'CPU\'s', 'Memory(MB)', 'HDD(GB)', 'Avg. Unixbench Score' ] collect_results(titles, results, results_dir, 'average-results') titles = ['VM', 'Timestamp (epoch)', 'Score'] results = list() for result in raw_results: machine_id, avg_score, scores = result for timestamp, score in scores: results.append([ '{} (id={})'.format(machines.get(machine_id, machine_id), machine_id), timestamp, score ]) collect_results(titles, results, results_dir, 'all-results') # report results with open(os.path.join(results_dir, 'parameters.md'), 'w') as params: params.write("# Parameters\n\n") for key, value in vars(options).items(): params.write("- **{}**: {}\n".format(key, value)) # pushing results to env_repo location = options.environment.split('.')[0] push_results_to_repo(results_dir, location)
def main(options): # Check dependencies if not os.path.exists(options.results_dir): print( "Not all dependencies are met. Make sure the result directory exists." ) return if not check_package('sshpass') or not check_package( 'python3-prettytable'): return # Prepare test run hostname = run_cmd_via_gevent('hostname').replace("\n", "") test_num = len(os.listdir('{}'.format(options.results_dir))) + 1 test_dir = "/" + datetime.datetime.today().strftime('%Y-%m-%d') test_dir += "_" + hostname + "_fio_testresults_{}".format(test_num) results_dir = options.results_dir + test_dir run_cmd_via_gevent('mkdir -p {}'.format(results_dir)) # list virtual and deployed cloudspaces vms = [] vms_index = set() j.clients.itsyouonline.get(data={ 'application_id_': options.application_id, 'secret_': options.secret }) ovc = j.clients.openvcloud.get(data={ 'address': options.environment, 'account': options.username }) cloudspaces_per_user = ovc.api.cloudapi.cloudspaces.list() cloudspaces_ids = options.cloudspaces if cloudspaces_ids: cloudspaces_ids = [int(_id) for _id in cloudspaces_ids.split(',')] cloudspaces_per_user = [ cs for cs in cloudspaces_per_user if cs['id'] in cloudspaces_ids ] for cs in cloudspaces_per_user: if cs['name'] == 'template_space': continue portforwards = ovc.api.cloudapi.portforwarding.list( cloudspaceId=cs['id']) for pi in portforwards: if 'machineId' not in pi or pi['machineId'] in vms_index: continue vms.append([pi['machineId'], pi['publicIp'], pi['publicPort']]) vms_index.add(pi['machineId']) if len(vms) < options.required_vms: print("Not enough vms available to run this test. {} < {}".format( len(vms), options.required_vms)) return vms = vms[:options.required_vms] # prepare test files = [ '{}/1_fio_vms/Machine_script.py'.format(options.testsuite), '{}/1_fio_vms/mount_disks.sh'.format(options.testsuite) ] pjobs = [ gevent.spawn(prepare_test, ovc, concurrency, options, files, *vm) for vm in vms ] gevent.joinall(pjobs) # mount disks if the filesystem will be used if options.type == "filesytem": gevent.joinall( [gevent.spawn(mount_disks, ovc, options, *vm) for vm in vms]) # run fio tests rjobs = [ gevent.spawn(fio_test, options, *job.value) for job in pjobs if job.value is not None ] gevent.joinall(rjobs) # collect results from machines rjobs = [ gevent.spawn(assemble_fio_test_results, results_dir, *job.value) for job in rjobs if job.value is not None ] gevent.joinall(rjobs) # collecting results in csv file with open(os.path.join(results_dir, 'parameters.md'), 'w') as params: params.write("# Parameters\n\n") for key, value in vars(options).items(): params.write("- **{}**: {}\n".format(key, value)) cwd = os.getcwd() j.sal.fs.copyFile( '{}/1_fio_vms/collect_results.py'.format(options.testsuite), results_dir) os.chdir(results_dir) j.sal.process.execute( 'python3 collect_results.py -dir {} -env {} -u {} -appid {} -secret {}' .format(results_dir, options.environment, options.username, options.application_id, options.secret)) # pushing results to env_repo os.chdir(cwd) location = options.environment.split('.')[0] push_results_to_repo(results_dir, location)
def main(options): from js9 import j # Check dependencies if not os.path.exists(options.results_dir): print( "Not all dependencies are met. Make sure the result directory exists." ) return if not check_package('sshpass') or not check_package( 'python3-prettytable'): return # Prepare test run hostname = run_cmd_via_gevent('hostname').replace("\n", "") test_num = len(os.listdir('{}'.format(options.results_dir))) + 1 test_dir = "/" + datetime.datetime.today().strftime('%Y-%m-%d') test_dir += "_" + hostname + "_pgbench_testresults_{}".format(test_num) results_dir = options.results_dir + test_dir run_cmd_via_gevent('mkdir -p {}'.format(results_dir)) # list virtual and deployed cloudspaces vms = [] vms_index = set() j.clients.itsyouonline.get(data={ 'application_id_': options.application_id, 'secret_': options.secret }) ovc = j.clients.openvcloud.get(data={ 'address': options.environment, 'account': options.username }) cloudspaces_per_user = ovc.api.cloudapi.cloudspaces.list() for cs in cloudspaces_per_user: if cs['name'] == 'template_space': continue portforwards = ovc.api.cloudapi.portforwarding.list( cloudspaceId=cs['id']) for pi in portforwards: if 'machineId' not in pi or pi['machineId'] in vms_index: continue vms.append([pi['machineId'], pi['publicIp'], pi['publicPort']]) vms_index.add(pi['machineId']) if len(vms) < options.required_vms: print("Not enough vms available to run this test. {} < {}".format( len(vms), options.required_vms)) return vms = vms[:options.required_vms] # prepare test files = ['{}/4_pgbench/run_pgbench.sh'.format(options.testsuite)] pjobs = [ gevent.spawn(prepare_test, ovc, concurrency, options, files, *vm) for vm in vms ] gevent.joinall(pjobs) # run pgbench tests rjobs = [ gevent.spawn(pgbench, options, *job.value) for job in pjobs if job.value is not None ] gevent.joinall(rjobs) # report results with open(os.path.join(results_dir, 'parameters.md'), 'w') as params: params.write("# Parameters\n\n") for key, value in vars(options).items(): params.write("- **{}**: {}\n".format(key, value)) total_iops = 0 with open(os.path.join(results_dir, 'results.csv'), 'w') as csvfile: fieldnames = ['machine_id', 'iops'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for job in rjobs: machine_id, iops = job.value if iops == 0: continue # skip machines whith errors writer.writerow({'machine_id': machine_id, 'iops': iops}) total_iops += iops writer.writerow({'machine_id': 'total iops', 'iops': total_iops}) print("==========================") print("Total iops: {}".format(total_iops)) print("==========================") # pushing results to env_repo location = options.environment.split('.')[0] push_results_to_repo(results_dir, location)