def RunBenchmarks(): """Runs all benchmarks in PerfKitBenchmarker. Returns: Exit status for the process. """ benchmark_specs = _CreateBenchmarkSpecs() if FLAGS.dry_run: print 'PKB will run with the following configurations:' for spec in benchmark_specs: print spec print '' return 0 collector = SampleCollector() try: tasks = [(RunBenchmarkTask, (spec, ), {}) for spec in benchmark_specs] if FLAGS.run_with_pdb and FLAGS.run_processes == 1: spec_sample_tuples = RunBenchmarkTasksInSeries(tasks) else: spec_sample_tuples = background_tasks.RunParallelProcesses( tasks, FLAGS.run_processes, FLAGS.run_processes_delay) benchmark_specs, sample_lists = zip(*spec_sample_tuples) for sample_list in sample_lists: collector.samples.extend(sample_list) finally: if collector.samples: collector.PublishSamples() if benchmark_specs: logging.info(benchmark_status.CreateSummary(benchmark_specs)) logging.info('Complete logs can be found at: %s', vm_util.PrependTempDir(LOG_FILE_NAME)) logging.info('Completion statuses can be found at: %s', vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME)) if stages.TEARDOWN not in FLAGS.run_stage: logging.info('To run again with this setup, please use --run_uri=%s', FLAGS.run_uri) if FLAGS.archive_bucket: archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket, gsutil_path=FLAGS.gsutil_path, prefix=FLAGS.run_uri + '_') # Write completion status file(s) completion_status_file_name = ( vm_util.PrependTempDir(COMPLETION_STATUS_FILE_NAME)) with open(completion_status_file_name, 'w') as status_file: _WriteCompletionStatusFile(benchmark_specs, status_file) if FLAGS.completion_status_file: with open(FLAGS.completion_status_file, 'w') as status_file: _WriteCompletionStatusFile(benchmark_specs, status_file) all_benchmarks_succeeded = all(spec.status == benchmark_status.SUCCEEDED for spec in benchmark_specs) return 0 if all_benchmarks_succeeded else 1
def RunIperf3UDPStream(sending_vm, receiving_vm, use_internal_ip=True): """Runs the Iperf3 UDP stream test. Args: sending_vm: The client VM that will send the UDP packets. receiving_vm: The server VM that will receive the UDP packets. use_internal_ip: if true, the private network will be used for the test. if false, the external network will be used for the test. Returns: List of sample objects each representing a single metric on a single run. """ iperf3_exec_dir = ntpath.join(sending_vm.temp_dir, IPERF3_DIR) def _RunIperf3UDP(vm, options): command = 'cd {iperf3_exec_dir}; .\\iperf3.exe {options}'.format( iperf3_exec_dir=iperf3_exec_dir, options=options) vm.RemoteCommand(command) receiver_ip = (receiving_vm.internal_ip if use_internal_ip else receiving_vm.ip_address) samples = [] for bandwidth in range(FLAGS.min_bandwidth_mb, FLAGS.max_bandwidth_mb, FLAGS.bandwidth_step_mb): sender_args = ('--client {server_ip} --udp -t {duration} -P {num_threads} ' '-b {bandwidth}M -l {buffer_len} > {out_file}'.format( server_ip=receiver_ip, duration=FLAGS.udp_stream_seconds, num_threads=FLAGS.udp_client_threads, bandwidth=bandwidth, buffer_len=FLAGS.udp_buffer_len, out_file=IPERF3_OUT_FILE)) # the "-1" flag will cause the server to exit after performing a single # test. This is necessary because the RemoteCommand call will not return # until the command completes, even if it is run as a daemon. receiver_args = '--server -1' process_args = [(_RunIperf3UDP, (receiving_vm, receiver_args), {}), (_RunIperf3UDP, (sending_vm, sender_args), {})] background_tasks.RunParallelProcesses(process_args, 200, 1) # retrieve the results and parse them cat_command = 'cd {iperf3_exec_dir}; cat {out_file}'.format( iperf3_exec_dir=iperf3_exec_dir, out_file=IPERF3_OUT_FILE) command_out, _ = sending_vm.RemoteCommand(cat_command) samples.extend( GetUDPStreamSamples(sending_vm, receiving_vm, command_out, bandwidth, use_internal_ip)) return samples
def testException(self): counter = Counter() calls = [(_IncrementCounter, (counter, ), {}), (_RaiseValueError, (), {}), (_IncrementCounter, (counter, ), {})] with self.assertRaises(errors.VmUtil.CalledProcessException): background_tasks.RunParallelProcesses(calls, max_concurrency=1) # RunParallelProcesses does not gurantee the tasks are run in order. self.assertLessEqual(counter.value, 2, 'Unexpected counter value')
def RunSingleBandwidth(bandwidth, sending_vm, receiving_vm, dest_ip, exec_path): """Create a server-client nuttcp pair. The server exits after the client completes its request. Args: bandwidth: the requested transmission bandwidth sending_vm: vm sending the UDP packets. receiving_vm: vm receiving the UDP packets. dest_ip: the IP of the receiver. exec_path: path to the nuttcp executable. Returns: output from the client nuttcp process. """ sender_args = ('-u -p{data_port} -P{control_port} -R{bandwidth} ' '-T{time} -l{packet_size} {dest_ip} > {out_file}').format( data_port=UDP_PORT, control_port=CONTROL_PORT, bandwidth=bandwidth, time=FLAGS.nuttcp_udp_stream_seconds, packet_size=FLAGS.nuttcp_udp_packet_size, dest_ip=dest_ip, out_file=NUTTCP_OUT_FILE) receiver_args = '-p{data_port} -P{control_port} -1'.format( data_port=UDP_PORT, control_port=CONTROL_PORT) # Process to run the nuttcp server server_process = multiprocessing.Process( name='server', target=_RunNuttcp, args=(receiving_vm, receiver_args, exec_path)) server_process.start() receiving_vm.WaitForProcessRunning('nuttcp', 30) # Process to run the nuttcp client client_process = multiprocessing.Process( name='client', target=_RunNuttcp, args=(sending_vm, sender_args, exec_path)) client_process.start() sending_vm.WaitForProcessRunning('nuttcp', 30) process_args = [ (_GetCpuUsage, (receiving_vm,), {}), (_GetCpuUsage, (sending_vm,), {})] background_tasks.RunParallelProcesses(process_args, 200) server_process.join() client_process.join()
def RunNtttcp(sending_vm, receiving_vm, receiving_ip_address, ip_type, udp, threads, time_s, packet_size, cooldown): """Run NTttcp and return the samples collected from the run.""" if cooldown: time.sleep(FLAGS.ntttcp_cooldown_time) # Clean up any stray ntttcp processes in case this is retry. _TaskKillNtttcp(sending_vm) _TaskKillNtttcp(receiving_vm) packet_size_string = '' if packet_size: packet_size_string = ' -l %d ' % packet_size shared_options = '-xml -t {time} -p {port} {packet_size}'.format( time=time_s, port=BASE_DATA_PORT, packet_size=packet_size_string) udp_string = '-u' if udp else '' sending_options = shared_options + ( '-s {udp} -m \'{threads},*,{ip}\' -rb {rb} -sb {sb}').format( udp=udp_string, threads=threads, ip=receiving_ip_address, rb=_GetSockBufferSize(FLAGS.ntttcp_sender_rb), sb=_GetSockBufferSize(FLAGS.ntttcp_sender_sb)) receiving_options = shared_options + ( '-r {udp} -m \'{threads},*,0.0.0.0\' -rb {rb} -sb {sb}').format( udp=udp_string, threads=threads, rb=_GetSockBufferSize(FLAGS.ntttcp_receiver_rb), sb=_GetSockBufferSize(FLAGS.ntttcp_receiver_sb)) # NTttcp will append to the xml file when it runs, which causes parsing # to fail if there was a preexisting xml file. To be safe, try deleting # the xml file. _RemoveXml(sending_vm) _RemoveXml(receiving_vm) process_args = [(_RunNtttcp, (sending_vm, sending_options), {}), (_RunNtttcp, (receiving_vm, receiving_options), {})] background_tasks.RunParallelProcesses(process_args, 200) sender_xml = _CatXml(sending_vm) receiver_xml = _CatXml(receiving_vm) metadata = {'ip_type': ip_type} for vm_specifier, vm in ('receiving', receiving_vm), ('sending', sending_vm): for k, v in six.iteritems(vm.GetResourceMetadata()): metadata['{0}_{1}'.format(vm_specifier, k)] = v return ParseNtttcpResults(sender_xml, receiver_xml, metadata)
def testException(self): manager = multiprocessing.managers.SyncManager() manager.start() lock = manager.Lock() counter = manager.Value('i', 0) calls = [(_IncrementCounter, (lock, counter), {}), (_RaiseValueError, (), {}), (_IncrementCounter, (lock, counter), {})] with self.assertRaises(errors.VmUtil.CalledProcessException): background_tasks.RunParallelProcesses(calls, max_concurrency=1) self.assertEqual(counter.value, 2)
def _RunDiskSpd(running_vm, access_pattern, diskspd_write_read_ratio, metadata): sending_options = _GenerateOption(access_pattern, diskspd_write_read_ratio) process_args = [(_RunDiskSpdWithOptions, (running_vm, sending_options), {}) ] background_tasks.RunParallelProcesses(process_args, 200) result_xml = _CatXml(running_vm) _RemoveTempFile(running_vm) _RemoveXml(running_vm) main_metric = 'ReadSpeed' if diskspd_write_read_ratio == 0 else 'WriteSpeed' return ParseDiskSpdResults(result_xml, metadata, main_metric)
def RunLatencyTest(sending_vm, receiving_vm, use_internal_ip=True): """Run the psping latency test. Uses a TCP request-response time to measure latency. Args: sending_vm: the vm to send the tcp request. receiving_vm: the vm acting as the server. use_internal_ip: whether or not to use the private IP or the public IP. Returns: list of samples representing latency between the two VMs. """ server_ip = (receiving_vm.internal_ip if use_internal_ip else receiving_vm.ip_address) client_command = ( 'cd {psping_exec_dir}; ' 'sleep 2;' # sleep to make sure the server starts first. '.\\psping.exe /accepteula -l {packet_size} -i 0 -q ' '-n {rr_count} -h {bucket_count} {ip}:{port}' ' > {out_file}').format(psping_exec_dir=sending_vm.temp_dir, packet_size=FLAGS.psping_packet_size, rr_count=FLAGS.psping_rr_count, bucket_count=FLAGS.psping_bucket_count, ip=server_ip, port=TEST_PORT, out_file=PSPING_OUTPUT_FILE) # PSPing does not have a configurable timeout. To get around this, start the # server as a background job, then kill it after 10 seconds server_command = ( '{psping_exec_dir}\\psping.exe /accepteula -s 0.0.0.0:{port};').format( psping_exec_dir=receiving_vm.temp_dir, port=TEST_PORT) process_args = [(_RunPsping, (receiving_vm, server_command), {}), (_RunPsping, (sending_vm, client_command), {})] background_tasks.RunParallelProcesses(process_args, 200, 1) cat_command = 'cd {psping_exec_dir}; cat {out_file}'.format( psping_exec_dir=sending_vm.temp_dir, out_file=PSPING_OUTPUT_FILE) output, _ = sending_vm.RemoteCommand(cat_command) return ParsePspingResults(output, sending_vm, receiving_vm, use_internal_ip)
def RunBenchmarks(): """Runs all benchmarks in PerfKitBenchmarker. Returns: Exit status for the process. """ benchmark_specs = _CreateBenchmarkSpecs() collector = SampleCollector() try: tasks = [(RunBenchmarkTask, (spec,), {}) for spec in benchmark_specs] spec_sample_tuples = background_tasks.RunParallelProcesses( tasks, FLAGS.run_processes) benchmark_specs, sample_lists = zip(*spec_sample_tuples) for sample_list in sample_lists: collector.samples.extend(sample_list) finally: if collector.samples: collector.PublishSamples() if benchmark_specs: logging.info(benchmark_status.CreateSummary(benchmark_specs)) logging.info('Complete logs can be found at: %s', vm_util.PrependTempDir(LOG_FILE_NAME)) if stages.TEARDOWN not in FLAGS.run_stage: logging.info( 'To run again with this setup, please use --run_uri=%s', FLAGS.run_uri) if FLAGS.archive_bucket: archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket, gsutil_path=FLAGS.gsutil_path, prefix=FLAGS.run_uri + '_') all_benchmarks_succeeded = all(spec.status == benchmark_status.SUCCEEDED for spec in benchmark_specs) return 0 if all_benchmarks_succeeded else 1
def RunNtttcp(sending_vm, receiving_vm, receiving_ip_address, ip_type): """Run NTttcp and return the samples collected from the run.""" packet_size_string = '' if FLAGS.ntttcp_packet_size: packet_size_string = ' -l %d ' % FLAGS.ntttcp_packet_size shared_options = '-xml -t {time} -p {port} {packet_size}'.format( time=FLAGS.ntttcp_time, port=BASE_DATA_PORT, packet_size=packet_size_string) udp_string = '-u' if FLAGS.ntttcp_udp else '' sending_options = shared_options + '-s {udp} -m \'{threads},*,{ip}\''.format( udp=udp_string, threads=FLAGS.ntttcp_threads, ip=receiving_ip_address) receiving_options = shared_options + ( '-r {udp} -m \'{threads},*,0.0.0.0\'').format( udp=udp_string, threads=FLAGS.ntttcp_threads) # NTttcp will append to the xml file when it runs, which causes parsing # to fail if there was a preexisting xml file. To be safe, try deleting # the xml file. _RemoveXml(sending_vm) _RemoveXml(receiving_vm) process_args = [(_RunNtttcp, (sending_vm, sending_options), {}), (_RunNtttcp, (receiving_vm, receiving_options), {})] background_tasks.RunParallelProcesses(process_args, 200) sender_xml = _CatXml(sending_vm) receiver_xml = _CatXml(receiving_vm) metadata = {'ip_type': ip_type} for vm_specifier, vm in ('receiving', receiving_vm), ('sending', sending_vm): for k, v in vm.GetResourceMetadata().iteritems(): metadata['{0}_{1}'.format(vm_specifier, k)] = v return ParseNtttcpResults(sender_xml, receiver_xml, metadata)
def RunDiskSpd(running_vm): """Run Diskspd and return the samples collected from the run.""" large_page_string = '-l' if FLAGS.diskspd_large_page else '' latency_stats_string = '-L' if FLAGS.diskspd_latency_stats else '' disable_affinity_string = '-n' if FLAGS.diskspd_disable_affinity else '' software_cache_string = '-Su' if FLAGS.diskspd_software_cache else '' write_through_string = '-Sw' if FLAGS.diskspd_write_through else '' block_size_string = str(FLAGS.diskspd_block_size) + \ str(FLAGS.diskspd_block_unit) access_pattern_string = str(FLAGS.diskspd_access_pattern) + \ str(FLAGS.diskspd_stride_or_alignment) + \ str(FLAGS.diskspd_stride_or_alignment_unit) throughput_per_ms_string = '' if FLAGS.diskspd_throughput_per_ms: throughput_per_ms_string = '-g' + str(FLAGS.diskspd_throughput_per_ms) sending_options = ('-c{filesize}K -d{duration} -t{threadcount} ' '-W{warmup} -C{cooldown} -Rxml -w{ratio} ' '{large_page} {latency_stats} {disable_affinity} ' '{software_cache} {write_through} {throughput}' '-b{block_size} -f{hint_string} -{access_pattern} ' '-o{outstanding_io} ' 'C:\\scratch\\{tempfile} > {xmlfile}').format( filesize=FLAGS.diskspd_file_size, duration=FLAGS.diskspd_duration, threadcount=FLAGS.diskspd_thread_number_per_file, warmup=FLAGS.diskspd_warmup, cooldown=FLAGS.diskspd_cooldown, ratio=FLAGS.diskspd_write_read_ratio, tempfile=DISKSPD_TMPFILE, xmlfile=DISKSPD_XMLFILE, large_page=large_page_string, latency_stats=latency_stats_string, disable_affinity=disable_affinity_string, software_cache=software_cache_string, write_through=write_through_string, access_pattern=access_pattern_string, block_size=block_size_string, hint_string=FLAGS.diskspd_access_hint, throughput=throughput_per_ms_string, outstanding_io=FLAGS.diskspd_outstanding_io) process_args = [(_RunDiskSpd, (running_vm, sending_options), {})] # run diskspd background_tasks.RunParallelProcesses(process_args, 200) result_xml = _CatXml(running_vm) _RemoveTempFile(running_vm) _RemoveXml(running_vm) metadata = {} for k, v in running_vm.GetResourceMetadata().iteritems(): metadata['{0}'.format(k)] = v # add the flag information to the metadata # some of the flags information has been included in the xml file metadata['diskspd_block_size'] = FLAGS.diskspd_block_size metadata['diskspd_block_size_unit'] = FLAGS.diskspd_block_unit metadata['diskspd_access_hint'] = FLAGS.diskspd_access_hint metadata['diskspd_access_pattern'] = FLAGS.diskspd_access_pattern metadata['diskspd_stride_or_alignment'] = FLAGS.diskspd_stride_or_alignment metadata['diskspd_stride_or_alignment_unit'] = FLAGS.diskspd_stride_or_alignment_unit metadata['diskspd_large_page'] = FLAGS.diskspd_large_page metadata['diskspd_latency_stats'] = FLAGS.diskspd_latency_stats metadata['diskspd_disable_affinity'] = FLAGS.diskspd_disable_affinity metadata['diskspd_write_through'] = FLAGS.diskspd_write_through metadata['diskspd_software_cache'] = FLAGS.diskspd_software_cache metadata['diskspd_outstanding_io'] = FLAGS.diskspd_outstanding_io metadata['diskspd_throughput'] = FLAGS.diskspd_throughput_per_ms return ParseDiskSpdResults(result_xml, metadata)
def testMoreThreadsThanConcurrencyLimit(self): calls = [(_ReturnArgs, ('a', ), {'b': i}) for i in range(10)] result = background_tasks.RunParallelProcesses(calls, max_concurrency=4) self.assertEqual(result, [(i, 'a') for i in range(10)])