def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ frontend = benchmark_spec.vm_groups['frontend'][0] backend = benchmark_spec.vm_groups['backend'][0] client = benchmark_spec.vm_groups['client'][0] def CleanupFrontend(vm): vm.RemoteCommand('sudo docker stop memcache_server') vm.RemoteCommand('sudo docker rm memcache_server') vm.RemoteCommand('sudo docker stop web_server') vm.RemoteCommand('sudo docker rm web_server') vm.RemoteCommand('sudo docker rmi cloudsuite/web-serving:memcached_server') vm.RemoteCommand('sudo docker rmi cloudsuite/web-serving:web_server') def CleanupBackend(vm): vm.RemoteCommand('sudo docker stop mysql_server') vm.RemoteCommand('sudo docker rm mysql_server') vm.RemoteCommand('sudo docker rmi cloudsuite/web-serving:db_server') def CleanupClient(vm): vm.RemoteCommand('sudo docker rm faban_client') vm.RemoteCommand('sudo docker rmi cloudsuite/web-serving:faban_client') target_arg_tuples = [(CleanupFrontend, [frontend], {}), (CleanupBackend, [backend], {}), (CleanupClient, [client], {})] vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def _RunIperf3ServerClientPair(sending_vm, sender_args, receiving_vm): """Create a server-client iperf3 pair. The server exits after the client completes its request. Args: sending_vm: The client VM that will send the UDP/TCP packets. sender_args: the client VM iperf3 args. receiving_vm: The server VM that will receive the UDP packets. Returns: output from the client iperf3 process. """ iperf3_exec_dir = ntpath.join(sending_vm.temp_dir, IPERF3_DIR) def _RunIperf3(vm, options): command = ('cd {iperf3_exec_dir}; ' '.\\iperf3.exe {options}').format( iperf3_exec_dir=iperf3_exec_dir, options=options) vm.RemoteCommand(command) receiver_args = '--server -1' threaded_args = [(_RunIperf3, (receiving_vm, receiver_args), {}), (_RunIperf3, (sending_vm, sender_args), {})] vm_util.RunParallelThreads(threaded_args, 200, 15) cat_command = 'cd {iperf3_exec_dir}; cat {out_file}'.format( iperf3_exec_dir=iperf3_exec_dir, out_file=IPERF3_OUT_FILE) command_out, _ = sending_vm.RemoteCommand(cat_command) return command_out
def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ server_seed = benchmark_spec.vm_groups['server_seed'][0] servers = benchmark_spec.vm_groups['servers'] client = benchmark_spec.vm_groups['client'][0] def CleanupServerCommon(vm): vm.RemoteCommand('sudo docker rm cassandra-server') vm.RemoteCommand('sudo docker rmi cloudsuite/data-serving:server') def CleanupServerSeed(vm): vm.RemoteCommand('sudo docker stop cassandra-server-seed') CleanupServerCommon(vm) def CleanupServer(vm): vm.RemoteCommand('sudo docker stop cassandra-server') CleanupServerCommon(vm) def CleanupClient(vm): vm.RemoteCommand('sudo docker rmi cloudsuite/data-serving:client') target_arg_tuples = ([(CleanupServerSeed, [server_seed], {})] + [(CleanupServer, [vm], {}) for vm in servers] + [(CleanupClient, [client], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ master = benchmark_spec.vm_groups['master'][0] workers = benchmark_spec.vm_groups['workers'] def CleanupCommon(vm): vm.RemoteCommand('sudo docker rm -v data') def CleanupMaster(vm): vm.RemoteCommand('sudo docker stop spark-master') vm.RemoteCommand('sudo docker rm spark-master') CleanupCommon(vm) def CleanupWorker(vm): vm.RemoteCommand('sudo docker stop spark-worker') vm.RemoteCommand('sudo docker rm spark-worker') CleanupCommon(vm) target_arg_tuples = ([(CleanupWorker, [vm], {}) for vm in workers] + [(CleanupMaster, [master], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Prepare(benchmark_spec): """Install docker. Pull images. Create datasets. Start Nginx server. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ server = benchmark_spec.vm_groups['server'][0] client = benchmark_spec.vm_groups['client'][0] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') vm.RemoteCommand('sudo docker pull cloudsuite/media-streaming:dataset') vm.RemoteCommand('sudo docker create --name dataset ' 'cloudsuite/media-streaming:dataset') def PrepareServer(vm): PrepareCommon(vm) vm.RemoteCommand('sudo docker pull cloudsuite/media-streaming:server') vm.RemoteCommand('sudo docker run -d --name server --net host ' '--volumes-from dataset ' 'cloudsuite/media-streaming:server') def PrepareClient(vm): PrepareCommon(vm) vm.RemoteCommand('sudo docker pull cloudsuite/media-streaming:client') target_arg_tuples = [(PrepareServer, [server], {}), (PrepareClient, [client], {})] vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ server = benchmark_spec.vm_groups['server'][0] client = benchmark_spec.vm_groups['client'][0] def CleanupCommon(vm): vm.RemoteCommand('sudo docker rm -v dataset') vm.RemoteCommand('sudo docker rmi cloudsuite/media-streaming:dataset') def CleanupServer(vm): server.RemoteCommand('sudo docker stop server') server.RemoteCommand('sudo docker rm server') server.RemoteCommand( 'sudo docker rmi cloudsuite/media-streaming:server') CleanupCommon(vm) def CleanupClient(vm): client.RemoteCommand( 'sudo docker rmi cloudsuite/media-streaming:client') CleanupCommon(vm) target_arg_tuples = [(CleanupServer, [server], {}), (CleanupClient, [client], {})] vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def WaitForBootCompletion(self): """Waits until VM is has booted.""" to_wait_for = [self._WaitForWinRmCommand] if FLAGS.cluster_boot_test_rdp_port_listening: to_wait_for.append(self._WaitForRdpPort) vm_util.RunParallelThreads([(method, [], {}) for method in to_wait_for], 2)
def RunNuttcp(sending_vm, receiving_vm, exec_path, dest_ip, network_type, iteration): """Run nuttcp tests. Args: sending_vm: vm sending the UDP packets. receiving_vm: vm receiving the UDP packets. exec_path: path to the nuttcp executable. dest_ip: the IP of the receiver. network_type: string representing the type of the network. iteration: the run number of the test. Returns: list of samples from the results of the nuttcp tests. """ def _RunNuttcp(vm, options): command = 'cd {exec_dir}; .\\{exec_path} {options}'.format( exec_dir=vm.temp_dir, exec_path=exec_path, options=options) vm.RemoteCommand(command) samples = [] bandwidths = [ '{b}m'.format(b=b) for b in xrange(FLAGS.nuttcp_min_bandwidth_mb, FLAGS.nuttcp_max_bandwidth_mb, FLAGS.nuttcp_bandwidth_step_mb) ] if FLAGS.nuttcp_udp_unlimited_bandwidth: bandwidths.append('u') for bandwidth in bandwidths: sender_args = ( '-u -p{data_port} -P{control_port} -R{bandwidth} ' '-T{time} -l{packet_size} {dest_ip} > {out_file}').format( data_port=UDP_PORT, control_port=CONTROL_PORT, bandwidth=bandwidth, time=FLAGS.nuttcp_udp_stream_seconds, packet_size=FLAGS.nuttcp_udp_packet_size, dest_ip=dest_ip, out_file=NUTTCP_OUT_FILE) receiver_args = '-p{data_port} -P{control_port} -1'.format( data_port=UDP_PORT, control_port=CONTROL_PORT) threaded_args = [(_RunNuttcp, (receiving_vm, receiver_args), {}), (_RunNuttcp, (sending_vm, sender_args), {})] vm_util.RunParallelThreads(threaded_args, 200, 5) # retrieve the results and parse them cat_command = 'cd {nuttcp_exec_dir}; cat {out_file}'.format( nuttcp_exec_dir=sending_vm.temp_dir, out_file=NUTTCP_OUT_FILE) command_out, _ = sending_vm.RemoteCommand(cat_command) samples.append( GetUDPStreamSample(command_out, sending_vm, receiving_vm, bandwidth, network_type, iteration)) return samples
def testException(self): int_list = [] calls = [(_AppendLength, (int_list,), {}), (_RaiseValueError, (), {}), (_AppendLength, (int_list,), {})] with self.assertRaises(errors.VmUtil.ThreadException): vm_util.RunParallelThreads(calls, max_concurrency=1) self.assertEqual(int_list, [0, 1])
def Prepare(benchmark_spec): """Install docker. Pull images. Start the master and slaves. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ master = benchmark_spec.vm_groups['master'][0] slaves = benchmark_spec.vm_groups['slaves'] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') def PrepareMaster(vm): PrepareCommon(vm) vm.RemoteCommand('sudo docker pull cloudsuite/data-analytics') vm.RemoteCommand('sudo docker run -d --name master --net host ' 'cloudsuite/data-analytics master') def PrepareSlave(vm): PrepareCommon(vm) vm.RemoteCommand('sudo docker pull cloudsuite/hadoop') vm.RemoteCommand('sudo docker run -d --name slave --net host ' 'cloudsuite/hadoop slave %s' % master.internal_ip) target_arg_tuples = ([(PrepareSlave, [vm], {}) for vm in slaves] + [(PrepareMaster, [master], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def RunSingleBandwidth(bandwidth): """Create a server-client nuttcp pair. The server exits after the client completes its request. Args: bandwidth: the requested transmission bandwidth Returns: output from the client nuttcp process. """ sender_args = ( '-u -p{data_port} -P{control_port} -R{bandwidth} ' '-T{time} -l{packet_size} {dest_ip} > {out_file}').format( data_port=UDP_PORT, control_port=CONTROL_PORT, bandwidth=bandwidth, time=FLAGS.nuttcp_udp_stream_seconds, packet_size=FLAGS.nuttcp_udp_packet_size, dest_ip=dest_ip, out_file=NUTTCP_OUT_FILE) receiver_args = '-p{data_port} -P{control_port} -1'.format( data_port=UDP_PORT, control_port=CONTROL_PORT) threaded_args = [(_RunNuttcp, (receiving_vm, receiver_args), {}), (_RunNuttcp, (sending_vm, sender_args), {})] vm_util.RunParallelThreads(threaded_args, 200, 5)
def _PerformFailoverTest(vm, metadata, benchmark_spec, sysbench_thread_count): """Runs the failover test - drive the workload while failing over.""" results = [] threaded_args = [ (_FailoverWorkloadThread, [vm, benchmark_spec, sysbench_thread_count, results, metadata], {}), (_FailOverThread, [benchmark_spec], {}) ] vm_util.RunParallelThreads(threaded_args, len(threaded_args), 0) return results
def RunSingleBandwidth(bandwidth, sending_vm, receiving_vm, dest_ip, exec_path): """Create a server-client nuttcp pair. The server exits after the client completes its request. Args: bandwidth: the requested transmission bandwidth sending_vm: vm sending the UDP packets. receiving_vm: vm receiving the UDP packets. dest_ip: the IP of the receiver. exec_path: path to the nuttcp executable. Returns: output from the client nuttcp process. """ sender_args = ('-u -p{data_port} -P{control_port} -R{bandwidth} ' '-T{time} -l{packet_size} {dest_ip} > {out_file}').format( data_port=UDP_PORT, control_port=CONTROL_PORT, bandwidth=bandwidth, time=FLAGS.nuttcp_udp_stream_seconds, packet_size=FLAGS.nuttcp_udp_packet_size, dest_ip=dest_ip, out_file=NUTTCP_OUT_FILE) receiver_args = '-p{data_port} -P{control_port} -1'.format( data_port=UDP_PORT, control_port=CONTROL_PORT) # Thread to run the nuttcp server server_thread = threading.Thread(name='server', target=_RunNuttcp, args=(receiving_vm, receiver_args, exec_path)) server_thread.start() receiving_vm.WaitForProcessRunning('nuttcp', 3) # Thread to run the nuttcp client client_thread = threading.Thread(name='client', target=_RunNuttcp, args=(sending_vm, sender_args, exec_path)) client_thread.start() sending_vm.WaitForProcessRunning('nuttcp', 3) threaded_args = [ (_GetCpuUsage, (receiving_vm, FLAGS.nuttcp_cpu_sample_time), {}), (_GetCpuUsage, (sending_vm, FLAGS.nuttcp_cpu_sample_time), {}) ] vm_util.RunParallelThreads(threaded_args, 200) server_thread.join() client_thread.join()
def RunIperf3UDPStream(sending_vm, receiving_vm, use_internal_ip=True): """Runs the Iperf3 UDP stream test. Args: sending_vm: The client VM that will send the UDP packets. receiving_vm: The server VM that will receive the UDP packets. use_internal_ip: if true, the private network will be used for the test. if false, the external network will be used for the test. Returns: List of sample objects each representing a single metric on a single run. """ iperf3_exec_dir = ntpath.join(sending_vm.temp_dir, IPERF3_DIR) def _RunIperf3(vm, options): command = 'cd {iperf3_exec_dir}; .\\iperf3.exe {options}'.format( iperf3_exec_dir=iperf3_exec_dir, options=options) vm.RemoteCommand(command) receiver_ip = (receiving_vm.internal_ip if use_internal_ip else receiving_vm.ip_address) samples = [] for bandwidth in xrange(FLAGS.min_bandwidth_mb, FLAGS.max_bandwidth_mb, FLAGS.bandwidth_step_mb): sender_args = ( '--client {server_ip} --udp -t {duration} -P {num_threads}' '-b {bandwidth}M -l {buffer_len} > {out_file}'.format( server_ip=receiver_ip, duration=FLAGS.udp_stream_seconds, num_threads=FLAGS.udp_client_threads, bandwidth=bandwidth, buffer_len=FLAGS.udp_buffer_len, out_file=IPERF3_OUT_FILE)) # the "-1" flag will cause the server to exit after performing a single # test. This is necessary because the RemoteCommand call will not return # until the command completes, even if it is run as a daemon. receiver_args = '--server -1' threaded_args = [(_RunIperf3, (receiving_vm, receiver_args), {}), (_RunIperf3, (sending_vm, sender_args), {})] vm_util.RunParallelThreads(threaded_args, 200, 15) # retrieve the results and parse them cat_command = 'cd {iperf3_exec_dir}; cat {out_file}'.format( iperf3_exec_dir=iperf3_exec_dir, out_file=IPERF3_OUT_FILE) command_out, _ = sending_vm.RemoteCommand(cat_command) samples.extend( GetUDPStreamSamples(sending_vm, receiving_vm, command_out, bandwidth, use_internal_ip)) return samples
def Prepare(benchmark_spec): """Install docker. Pull images. Start nginx, mysql, and memcached. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ frontend = benchmark_spec.vm_groups['frontend'][0] backend = benchmark_spec.vm_groups['backend'][0] client = benchmark_spec.vm_groups['client'][0] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') vm.RemoteCommand("sudo sh -c 'echo %s web_server >>/etc/hosts'" % frontend.internal_ip) vm.RemoteCommand("sudo sh -c 'echo %s memcache_server >>/etc/hosts'" % frontend.internal_ip) vm.RemoteCommand("sudo sh -c 'echo %s mysql_server >>/etc/hosts'" % backend.internal_ip) vm.RemoteCommand("sudo sh -c 'echo %s faban_client >>/etc/hosts'" % client.internal_ip) def PrepareFrontend(vm): PrepareCommon(vm) vm.Install('cloudsuite/web-serving:web_server') vm.Install('cloudsuite/web-serving:memcached_server') vm.RemoteCommand('sudo docker run -dt --net host --name web_server ' 'cloudsuite/web-serving:web_server ' '/etc/bootstrap.sh mysql_server memcache_server %s' % (FLAGS.cloudsuite_web_serving_pm_max_children)) vm.RemoteCommand( 'sudo docker run -dt --net host --name memcache_server ' 'cloudsuite/web-serving:memcached_server') def PrepareBackend(vm): PrepareCommon(vm) vm.Install('cloudsuite/web-serving:db_server') vm.RemoteCommand('sudo docker run -dt --net host --name mysql_server ' 'cloudsuite/web-serving:db_server web_server') def PrepareClient(vm): PrepareCommon(vm) vm.Install('cloudsuite/web-serving:faban_client') target_arg_tuples = [(PrepareFrontend, [frontend], {}), (PrepareBackend, [backend], {}), (PrepareClient, [client], {})] vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def RunLatencyTest(sending_vm, receiving_vm, use_internal_ip=True): """Run the psping latency test. Uses a TCP request-response time to measure latency. Args: sending_vm: the vm to send the tcp request. receiving_vm: the vm acting as the server. use_internal_ip: whether or not to use the private IP or the public IP. Returns: list of samples representing latency between the two VMs. """ server_ip = (receiving_vm.internal_ip if use_internal_ip else receiving_vm.ip_address) client_command = ( 'cd {psping_exec_dir}; ' 'sleep 2;' # sleep to make sure the server starts first. '.\\psping.exe /accepteula -l {packet_size} -i 0 -q ' '-n {rr_count} -h {bucket_count} {ip}:{port}' ' > {out_file}').format(psping_exec_dir=sending_vm.temp_dir, packet_size=FLAGS.psping_packet_size, rr_count=FLAGS.psping_rr_count, bucket_count=FLAGS.psping_bucket_count, ip=server_ip, port=TEST_PORT, out_file=PSPING_OUTPUT_FILE) # PSPing does not have a configurable timeout. To get around this, start the # server as a background job, then kill it after 10 seconds server_command = ( '{psping_exec_dir}\\psping.exe /accepteula -s 0.0.0.0:{port};').format( psping_exec_dir=sending_vm.temp_dir, port=TEST_PORT) threaded_args = [(_RunPsping, (receiving_vm, server_command), {}), (_RunPsping, (sending_vm, client_command), {})] vm_util.RunParallelThreads(threaded_args, 200, 1) cat_command = 'cd {psping_exec_dir}; cat {out_file}'.format( psping_exec_dir=sending_vm.temp_dir, out_file=PSPING_OUTPUT_FILE) output, _ = sending_vm.RemoteCommand(cat_command) return ParsePspingResults(output, sending_vm, receiving_vm, use_internal_ip)
def Prepare(benchmark_spec): """Install docker. Pull the required images from DockerHub. Start Solr index node and client. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ servers = benchmark_spec.vm_groups['servers'][0] clients = benchmark_spec.vm_groups['clients'] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') def PrepareServer(vm): PrepareCommon(vm) server_cmd = ('sudo echo \'DOCKER_OPTS="-g %s"\'' '| sudo tee /etc/default/docker > /dev/null' % (DISK_PATH)) stdout, _ = vm.RemoteCommand(server_cmd, should_log=True) server_cmd = 'sudo service docker restart' stdout, _ = vm.RemoteCommand(server_cmd, should_log=True) vm.Install('cloudsuite/web-search:server') server_cmd = ('sudo docker run -d --net host ' '--name server cloudsuite/web-search:server %s 1' % (FLAGS.cloudsuite_web_search_server_heap_size)) stdout, _ = servers.RemoteCommand(server_cmd, should_log=True) def PrepareClient(vm): PrepareCommon(vm) vm.Install('cloudsuite/web-search:client') PrepareServer(servers) target_arg_tuples = ([(PrepareClient, [vm], {}) for vm in clients]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Prepare(benchmark_spec): """Prepare docker containers and set the dataset up. Install docker. Pull the required images from DockerHub. Create a table into server-seed and load the dataset. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ server_seed = benchmark_spec.vm_groups['server_seed'][0] servers = benchmark_spec.vm_groups['servers'] client = benchmark_spec.vm_groups['client'][0] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') def PrepareServerSeed(vm): PrepareCommon(vm) vm.Install('cloudsuite/data-serving:server') vm.RemoteCommand('sudo docker run -d --name cassandra-server-seed ' '--net host cloudsuite/data-serving:server') def PrepareServer(vm): PrepareCommon(vm) vm.Install('cloudsuite/data-serving:server') start_server_cmd = ('sudo docker run -d --name cassandra-server ' '-e CASSANDRA_SEEDS=%s --net host ' 'cloudsuite/data-serving:server' % server_seed.internal_ip) vm.RemoteCommand(start_server_cmd) def PrepareClient(vm): PrepareCommon(vm) vm.Install('cloudsuite/data-serving:client') target_arg_tuples = ([(PrepareServerSeed, [server_seed], {})] + [(PrepareServer, [vm], {}) for vm in servers] + [(PrepareClient, [client], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ servers = benchmark_spec.vm_groups['servers'][0] clients = benchmark_spec.vm_groups['clients'] def CleanupClient(vm): vm.RemoteCommand('sudo docker stop client') vm.RemoteCommand('sudo docker rm client') def CleanupServer(vm): vm.RemoteCommand('sudo docker stop server') vm.RemoteCommand('sudo docker rm server') target_arg_tuples = ([(CleanupClient, [vm], {}) for vm in clients] + [(CleanupServer, [servers], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Prepare(benchmark_spec): """Install docker. Pull the required images from DockerHub, create datasets, and start Spark master and workers. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ master = benchmark_spec.vm_groups['master'][0] workers = benchmark_spec.vm_groups['workers'] def PrepareCommon(vm): if not docker.IsInstalled(vm): vm.Install('docker') vm.RemoteCommand('sudo docker pull cloudsuite/spark') vm.RemoteCommand('sudo docker pull cloudsuite/twitter-dataset-graph') vm.RemoteCommand('sudo docker create --name data ' 'cloudsuite/twitter-dataset-graph') def PrepareMaster(vm): PrepareCommon(vm) vm.RemoteCommand('sudo docker pull cloudsuite/graph-analytics') master_cmd = ('sudo docker run -d --net host -e SPARK_MASTER_IP=%s ' '--name spark-master cloudsuite/spark master' % vm.internal_ip) vm.RemoteCommand(master_cmd) def PrepareWorker(vm): PrepareCommon(vm) worker_cmd = ('sudo docker run -d --net host --volumes-from data ' '--name spark-worker cloudsuite/spark worker ' 'spark://%s:7077' % master.internal_ip) vm.RemoteCommand(worker_cmd) target_arg_tuples = ([(PrepareWorker, [vm], {}) for vm in workers] + [(PrepareMaster, [master], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def Cleanup(benchmark_spec): """Stop and remove docker containers. Remove images. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. """ master = benchmark_spec.vm_groups['master'][0] slaves = benchmark_spec.vm_groups['slaves'] def CleanupMaster(vm): vm.RemoteCommand('sudo docker stop master') vm.RemoteCommand('sudo docker rm master') vm.RemoteCommand('sudo docker rmi cloudsuite/data-analytics') def CleanupSlave(vm): vm.RemoteCommand('sudo docker stop slave') vm.RemoteCommand('sudo docker rm slave') vm.RemoteCommand('sudo docker rmi cloudsuite/hadoop') target_arg_tuples = ([(CleanupSlave, [vm], {}) for vm in slaves] + [(CleanupMaster, [master], {})]) vm_util.RunParallelThreads(target_arg_tuples, len(target_arg_tuples))
def RunGetLatencyAtCpu(cloud_instance, client_vms): """Run a modified binary search to find latency at a given CPU. Args: cloud_instance: A managed cloud instance. Only works on managed cloud instances but could extend to vms. client_vms: Need at least two client vms, one to hold the CPU utilization load and the other to get the single threaded latency. Returns: A list of sample.Sample instances. """ samples = [] server_ip = cloud_instance.GetMemoryStoreIp() server_port = cloud_instance.GetMemoryStorePort() password = cloud_instance.GetMemoryStorePassword() load_vm = client_vms[0] latency_measurement_vm = client_vms[-1] # Implement modified binary search to find optimal client count +/- tolerance # of target CPU usage target = MEMTIER_CPU_TARGET.value # Larger clusters need two threads to get to maximum CPU threads = 1 if cloud_instance.GetInstanceSize( ) < 150 or target < 0.5 else 2 pipeline = 1 # Set maximum of the binary search based off size of the instance upper_bound = cloud_instance.GetInstanceSize() // 2 + 10 lower_bound = 1 current_clients = 1 while lower_bound < upper_bound: current_clients = (upper_bound + lower_bound) // 2 _Run(vm=load_vm, server_ip=server_ip, server_port=server_port, threads=threads, pipeline=pipeline, clients=current_clients, password=password) cpu_percent = cloud_instance.MeasureCpuUtilization( MEMTIER_CPU_DURATION.value) if not cpu_percent: raise errors.Benchmarks.RunError( 'Could not measure CPU utilization for the instance.') logging.info( 'Tried %s clients and got %s%% CPU utilization for the last run with ' 'the target CPU being %s%%', current_clients, cpu_percent, target) if cpu_percent < target - CPU_TOLERANCE: lower_bound = current_clients + 1 elif cpu_percent > target + CPU_TOLERANCE: upper_bound = current_clients - 1 else: logging.info( 'Finished binary search and the current client count is %s', current_clients) process_args = [ (_Run, [ load_vm, server_ip, server_port, threads, pipeline, current_clients, password ], {}), (_GetSingleThreadedLatency, [latency_measurement_vm, server_ip, server_port, password], {}) ] results = vm_util.RunParallelThreads(process_args, len(process_args)) metadata = GetMetadata(clients=current_clients, threads=threads, pipeline=pipeline) metadata[ 'measured_cpu_percent'] = cloud_instance.MeasureCpuUtilization( MEMTIER_CPU_DURATION.value) samples.extend(results[1].GetSamples(metadata)) return samples # If complete binary search without finding a client count, # it's not possible on this configuration. raise errors.Benchmarks.RunError( 'Completed binary search and did not find a client count that worked for ' 'this configuration and CPU utilization.')
def StopBackgroundWorkload(self): targets = [(vm.StopBackgroundWorkload, (), {}) for vm in self.vms] vm_util.RunParallelThreads(targets, len(targets))
def testMoreThreadsThanConcurrencyLimit(self): calls = [(_ReturnArgs, ('a',), {'b': i}) for i in range(10)] result = vm_util.RunParallelThreads(calls, max_concurrency=4) self.assertEqual(result, [(i, 'a') for i in range(10)])