def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ remote_path = remote_path or '~/' # In order to expand "~" and "~user" we use ntpath.expanduser(), # but it relies on environment variables being set. This modifies # the HOME environment variable in order to use that function, and then # restores it to its previous value. home = os.environ.get('HOME') try: os.environ['HOME'] = self.home_dir remote_path = ntpath.expanduser(remote_path) finally: if home is None: del os.environ['HOME'] else: os.environ['HOME'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RemoteHostCopy(self, file_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: file_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ if vm_util.RunningOnWindows(): if ':' in file_path: # scp doesn't like colons in paths. file_path = file_path.split(':', 1)[1] # Replace the last instance of '\' with '/' to make scp happy. file_path = '/'.join(file_path.rsplit('\\', 1)) remote_location = '%s@%s:%s' % (self.user_name, self.ip_address, remote_path) scp_cmd = ['scp', '-P', str(self.ssh_port), '-pr'] scp_cmd.extend(vm_util.GetSshOptions(self.ssh_private_key)) if copy_to: scp_cmd.extend([file_path, remote_location]) else: scp_cmd.extend([remote_location, file_path]) stdout, stderr, retcode = vm_util.IssueCommand(scp_cmd, timeout=None) if retcode: full_cmd = ' '.join(scp_cmd) error_text = ('Got non-zero return code (%s) executing %s\n' 'STDOUT: %sSTDERR: %s' % (retcode, full_cmd, stdout, stderr)) raise errors.VirtualMachine.RemoteCommandError(error_text)
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ remote_path = remote_path or '~/' home = os.environ['HOME'] try: os.environ['HOME'] = self.home_dir remote_path = ntpath.expanduser(remote_path) finally: os.environ['HOME'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.ip_address, remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def SetUpPKB(): """Set globals and environment variables for PKB. After SetUpPKB() returns, it should be possible to call PKB functions, like benchmark_spec.Prepare() or benchmark_spec.Run(). SetUpPKB() also modifies the local file system by creating a temp directory and storing new SSH keys. """ try: _InitializeRunUri() except errors.Error as e: logging.error(e) sys.exit(1) # Initialize logging. vm_util.GenTempDir() log_util.ConfigureLogging( stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level], log_path=vm_util.PrependTempDir(LOG_FILE_NAME), run_uri=FLAGS.run_uri, file_log_level=log_util.LOG_LEVELS[FLAGS.file_log_level]) logging.info('PerfKitBenchmarker version: %s', version.VERSION) # Translate deprecated flags and log all provided flag values. disk.WarnAndTranslateDiskFlags() _LogCommandLineFlags() # Check environment. if not FLAGS.ignore_package_requirements: requirements.CheckBasicRequirements() if FLAGS.os_type == os_types.WINDOWS and not vm_util.RunningOnWindows(): logging.error('In order to run benchmarks on Windows VMs, you must be ' 'running on Windows.') sys.exit(1) for executable in REQUIRED_EXECUTABLES: if not vm_util.ExecutableOnPath(executable): raise errors.Setup.MissingExecutableError( 'Could not find required executable "%s"', executable) vm_util.SSHKeyGen() if FLAGS.static_vm_file: with open(FLAGS.static_vm_file) as fp: static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile( fp) events.initialization_complete.send(parsed_flags=FLAGS)
def RemoteCopy(self, local_path, remote_path='', copy_to=True): """Copies a file to or from the VM. Args: local_path: Local path to file. remote_path: Optional path of where to copy file on remote host. copy_to: True to copy to vm, False to copy from vm. Raises: RemoteCommandError: If there was a problem copying the file. """ # Join with '' to get a trailing \ remote_path = remote_path or ntpath.join(self.home_dir, '') # In order to expand "~" and "~user" we use ntpath.expanduser(), # but it relies on environment variables being set. This modifies # the USERPROFILE environment variable in order to use that function, and # then restores it to its previous value. home = os.environ.get('USERPROFILE') try: os.environ['USERPROFILE'] = self.home_dir remote_path = ntpath.expanduser(remote_path) # Some Windows path's like C:\Users\ADMINI~1 contain ~, # but they should not start with ~ for dir_part in remote_path.split(ntpath.sep): assert not dir_part.startswith( '~'), f'Failed to expand {remote_path}' finally: if home is None: del os.environ['USERPROFILE'] else: os.environ['USERPROFILE'] = home drive, remote_path = ntpath.splitdrive(remote_path) remote_drive = (drive or self.system_drive).rstrip(':') network_drive = '\\\\%s\\%s$' % (self.GetConnectionIp(), remote_drive) if vm_util.RunningOnWindows(): self._PsDriveRemoteCopy(local_path, remote_path, copy_to, network_drive) else: self._SmbclientRemoteCopy(local_path, remote_path, copy_to, network_drive)
def RunBenchmarks(publish=True): """Runs all benchmarks in PerfKitBenchmarker. Args: publish: A boolean indicating whether results should be published. Returns: Exit status for the process. """ if FLAGS.version: print version.VERSION return for executable in REQUIRED_EXECUTABLES: if not vm_util.ExecutableOnPath(executable): logging.error('Could not find required executable "%s".' % executable) return 1 if FLAGS.run_uri is None: if FLAGS.run_stage not in [STAGE_ALL, STAGE_PREPARE]: # Attempt to get the last modified run directory. run_uri = vm_util.GetLastRunUri() if run_uri: FLAGS.run_uri = run_uri logging.warning( 'No run_uri specified. Attempting to run "%s" with --run_uri=%s.', FLAGS.run_stage, FLAGS.run_uri) else: logging.error('No run_uri specified. Could not run "%s".', FLAGS.run_stage) return 1 else: FLAGS.run_uri = str(uuid.uuid4())[-8:] elif not FLAGS.run_uri.isalnum() or len( FLAGS.run_uri) > MAX_RUN_URI_LENGTH: logging.error('run_uri must be alphanumeric and less than or equal ' 'to 10 characters in length.') return 1 vm_util.GenTempDir() log_util.ConfigureLogging( stderr_log_level=log_util.LOG_LEVELS[FLAGS.log_level], log_path=vm_util.PrependTempDir(LOG_FILE_NAME), run_uri=FLAGS.run_uri) _LogCommandLineFlags() if FLAGS.os_type == benchmark_spec.WINDOWS and not vm_util.RunningOnWindows( ): logging.error('In order to run benchmarks on Windows VMs, you must be ' 'running on Windows.') return 1 vm_util.SSHKeyGen() collector = SampleCollector() events.initialization_complete.send(parsed_flags=FLAGS) if FLAGS.static_vm_file: with open(FLAGS.static_vm_file) as fp: static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile( fp) if FLAGS.benchmark_config_pair: # Convert benchmark_config_pair into a {benchmark_name: file_name} # dictionary. tmp_dict = {} for config_pair in FLAGS.benchmark_config_pair: pair = config_pair.split(':') tmp_dict[pair[0]] = pair[1] FLAGS.benchmark_config_pair = tmp_dict try: benchmark_list = benchmark_sets.GetBenchmarksFromFlags() total_benchmarks = len(benchmark_list) if FLAGS.parallelism > 1: sequence_range = range(total_benchmarks, 0, -1) args = [((benchmark, collector, sequence_counter, total_benchmarks), {}) for benchmark, sequence_counter in zip( benchmark_list, sequence_range)] vm_util.RunThreaded(RunBenchmark, args, max_concurrent_threads=FLAGS.parallelism) else: sequence_range = range(1, total_benchmarks + 1) for benchmark, sequence_counter in zip(benchmark_list, sequence_range): RunBenchmark(benchmark, collector, sequence_counter, total_benchmarks) finally: if collector.samples: collector.PublishSamples() logging.info('Complete logs can be found at: %s', vm_util.PrependTempDir(LOG_FILE_NAME)) if FLAGS.run_stage not in [STAGE_ALL, STAGE_CLEANUP]: logging.info('To run again with this setup, please use --run_uri=%s', FLAGS.run_uri)
def RemoteHostCommand(self, command, should_log=False, retries=SSH_RETRIES, ignore_failure=False, login_shell=False, suppress_warning=False, timeout=None): """Runs a command on the VM. This is guaranteed to run on the host VM, whereas RemoteCommand might run within i.e. a container in the host VM. Args: command: A valid bash command. should_log: A boolean indicating whether the command result should be logged at the info level. Even if it is false, the results will still be logged at the debug level. retries: The maximum number of times RemoteCommand should retry SSHing when it receives a 255 return code. ignore_failure: Ignore any failure if set to true. login_shell: Run command in a login shell. suppress_warning: Suppress the result logging from IssueCommand when the return code is non-zero. Returns: A tuple of stdout and stderr from running the command. Raises: RemoteCommandError: If there was a problem establishing the connection. """ if vm_util.RunningOnWindows(): # Multi-line commands passed to ssh won't work on Windows unless the # newlines are escaped. command = command.replace('\n', '\\n') user_host = '%s@%s' % (self.user_name, self.ip_address) ssh_cmd = ['ssh', '-A', '-p', str(self.ssh_port), user_host] ssh_cmd.extend(vm_util.GetSshOptions(self.ssh_private_key)) try: if login_shell: ssh_cmd.extend(['-t', '-t', 'bash -l -c "%s"' % command]) self._pseudo_tty_lock.acquire() else: ssh_cmd.append(command) for _ in range(retries): stdout, stderr, retcode = vm_util.IssueCommand( ssh_cmd, force_info_log=should_log, suppress_warning=suppress_warning, timeout=timeout) if retcode != 255: # Retry on 255 because this indicates an SSH failure break finally: if login_shell: self._pseudo_tty_lock.release() if retcode: full_cmd = ' '.join(ssh_cmd) error_text = ('Got non-zero return code (%s) executing %s\n' 'Full command: %s\nSTDOUT: %sSTDERR: %s' % (retcode, command, full_cmd, stdout, stderr)) if not ignore_failure: raise errors.VirtualMachine.RemoteCommandError(error_text) return stdout, stderr
def RunBenchmarks(publish=True): """Runs all benchmarks in PerfKitBenchmarker. Args: publish: A boolean indicating whether results should be published. Returns: Exit status for the process. """ if FLAGS.version: print version.VERSION return _LogCommandLineFlags() if FLAGS.os_type == benchmark_spec.WINDOWS and not vm_util.RunningOnWindows( ): logging.error('In order to run benchmarks on Windows VMs, you must be ' 'running on Windows.') return 1 collector = SampleCollector() if FLAGS.static_vm_file: with open(FLAGS.static_vm_file) as fp: static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile( fp) run_status_lists = [] benchmark_tuple_list = benchmark_sets.GetBenchmarksFromFlags() total_benchmarks = len(benchmark_tuple_list) benchmark_counts = collections.defaultdict(itertools.count) args = [] for i, benchmark_tuple in enumerate(benchmark_tuple_list): benchmark_module, user_config = benchmark_tuple benchmark_name = benchmark_module.BENCHMARK_NAME benchmark_uid = benchmark_name + str( benchmark_counts[benchmark_name].next()) run_status_lists.append( [benchmark_name, benchmark_uid, benchmark_status.SKIPPED]) args.append((benchmark_module, collector, i + 1, total_benchmarks, benchmark_module.GetConfig(user_config), benchmark_uid)) try: for run_args, run_status_list in zip(args, run_status_lists): benchmark_module, _, sequence_number, _, _, benchmark_uid = run_args benchmark_name = benchmark_module.BENCHMARK_NAME try: run_status_list[2] = benchmark_status.FAILED RunBenchmark(*run_args) run_status_list[2] = benchmark_status.SUCCEEDED except BaseException as e: msg = 'Benchmark {0}/{1} {2} (UID: {3}) failed.'.format( sequence_number, total_benchmarks, benchmark_name, benchmark_uid) if (isinstance(e, KeyboardInterrupt) or FLAGS.stop_after_benchmark_failure): logging.error('%s Execution will not continue.', msg) break else: logging.error('%s Execution will continue.', msg) finally: if collector.samples: collector.PublishSamples() if run_status_lists: logging.info(benchmark_status.CreateSummary(run_status_lists)) logging.info('Complete logs can be found at: %s', vm_util.PrependTempDir(LOG_FILE_NAME)) if FLAGS.run_stage not in [STAGE_ALL, STAGE_TEARDOWN]: logging.info('To run again with this setup, please use --run_uri=%s', FLAGS.run_uri) if FLAGS.archive_bucket: archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket, gsutil_path=FLAGS.gsutil_path, prefix=FLAGS.run_uri + '_') all_benchmarks_succeeded = all(r[2] == benchmark_status.SUCCEEDED for r in run_status_lists) return 0 if all_benchmarks_succeeded else 1
def RunBenchmarks(): """Runs all benchmarks in PerfKitBenchmarker. Returns: Exit status for the process. """ if FLAGS.version: print version.VERSION return _LogCommandLineFlags() if FLAGS.os_type == os_types.WINDOWS and not vm_util.RunningOnWindows(): logging.error('In order to run benchmarks on Windows VMs, you must be ' 'running on Windows.') return 1 collector = SampleCollector() if FLAGS.static_vm_file: with open(FLAGS.static_vm_file) as fp: static_virtual_machine.StaticVirtualMachine.ReadStaticVirtualMachineFile( fp) benchmark_run_list = _CreateBenchmarkRunList() try: for run_args, run_status_list in benchmark_run_list: benchmark_module, sequence_number, _, _, benchmark_uid = run_args benchmark_name = benchmark_module.BENCHMARK_NAME try: run_status_list[2] = benchmark_status.FAILED RunBenchmark(*run_args, collector=collector) run_status_list[2] = benchmark_status.SUCCEEDED except BaseException as e: msg = 'Benchmark {0}/{1} {2} (UID: {3}) failed.'.format( sequence_number, len(benchmark_run_list), benchmark_name, benchmark_uid) if (isinstance(e, KeyboardInterrupt) or FLAGS.stop_after_benchmark_failure): logging.error('%s Execution will not continue.', msg) break else: logging.error('%s Execution will continue.', msg) finally: if collector.samples: collector.PublishSamples() if benchmark_run_list: run_status_lists = tuple(r for _, r in benchmark_run_list) logging.info(benchmark_status.CreateSummary(run_status_lists)) logging.info('Complete logs can be found at: %s', vm_util.PrependTempDir(LOG_FILE_NAME)) if stages.TEARDOWN not in FLAGS.run_stage: logging.info('To run again with this setup, please use --run_uri=%s', FLAGS.run_uri) if FLAGS.archive_bucket: archive.ArchiveRun(vm_util.GetTempDir(), FLAGS.archive_bucket, gsutil_path=FLAGS.gsutil_path, prefix=FLAGS.run_uri + '_') all_benchmarks_succeeded = all(r[2] == benchmark_status.SUCCEEDED for _, r in benchmark_run_list) return 0 if all_benchmarks_succeeded else 1