def get_xfer_rate(src, dst): """Compute transfer rate from src to dst as Mbit/sec Execute a copy from |src| to |dst| and returns the file copy transfer rate in Mbit/sec @param src, dst: paths for source and destination @return trasfer rate (float) in Mbit/sec """ assert os.path.isfile(src) assert os.path.isfile(dst) utils.drop_caches() start = datetime.datetime.now() utils.force_copy(src, dst) end = datetime.datetime.now() delta = end - start # compute seconds (as float) from microsecs delta_secs = delta.seconds + (delta.microseconds / USECS_IN_SEC) # compute Mbit from bytes size_Mbit = (os.path.getsize(src) * 8.0) / (1000 * 1000) logging.info( 'file trasferred: size (Mbits): %f, start: %f, end: %d,' ' delta (secs): %f', size_Mbit, start.second + start.microsecond / USECS_IN_SEC, end.second + end.microsecond / USECS_IN_SEC, delta_secs) # return the xfer rate in Mbits/secs having bytes/microsec return size_Mbit / delta_secs
def wrapped(self, *args, **dargs): try: return f(self, *args, **dargs) finally: if self._logger.global_filename == 'status': self.harness.run_test_complete() if self.drop_caches: utils.drop_caches()
def wrapped(self, *args, **dargs): try: return f(self, *args, **dargs) finally: if self._logger.global_filename == 'status': self.harness.run_test_complete() if self.drop_caches: utils.drop_caches()
def _init_drop_caches(self, drop_caches): """ Perform the drop caches initialization. """ self.drop_caches_between_iterations = True self.drop_caches = drop_caches if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches()
def wrapped(self, *args, **dargs): try: return f(self, *args, **dargs) finally: if self.log_filename == self.DEFAULT_LOG_FILENAME: self.harness.run_test_complete() if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches()
def wrapped(self, *args, **dargs): try: return f(self, *args, **dargs) finally: if self._logger.global_filename == "status": self.harness.run_test_complete() if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches()
def _init_drop_caches(self, drop_caches): """ Perform the drop caches initialization. """ self.drop_caches_between_iterations = ( global_config.global_config.get_config_value('CLIENT', 'drop_caches_between_iterations', type=bool, default=True)) self.drop_caches = drop_caches if self.drop_caches: utils.drop_caches()
def _init_drop_caches(self, drop_caches): """ Perform the drop caches initialization. """ self.drop_caches_between_iterations = global_config.global_config.get_config_value( "CLIENT", "drop_caches_between_iterations", type=bool, default=True ) self.drop_caches = drop_caches if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches()
def _init_drop_caches(self, drop_caches): """ Perform the drop caches initialization. """ self.drop_caches_between_iterations = ( GLOBAL_CONFIG.get_config_value('CLIENT', 'drop_caches_between_iterations', type=bool, default=True)) self.drop_caches = drop_caches if self.drop_caches: utils.drop_caches()
def _init_drop_caches(self, drop_caches): """ Perform the drop caches initialization. """ self.drop_caches_between_iterations = ( global_config.global_config.get_config_value( 'CLIENT', 'drop_caches_between_iterations', type=bool, default=True)) self.drop_caches = drop_caches if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches()
def drop_caches_between_iterations(self): if self.job.drop_caches_between_iterations: utils.drop_caches()
def __init__(self, control, options, drop_caches=True, extra_copy_cmdline=None): """ Prepare a client side job object. @param control: The control file (pathname of). @param options: an object which includes: jobtag: The job tag string (eg "default"). cont: If this is the continuation of this job. harness_type: An alternative server harness. [None] use_external_logging: If true, the enable_external_logging method will be called during construction. [False] @param drop_caches: If true, utils.drop_caches() is called before and between all tests. [True] @param extra_copy_cmdline: list of additional /proc/cmdline arguments to copy from the running kernel to all the installed kernels with this job """ self.autodir = os.environ['AUTODIR'] self.bindir = os.path.join(self.autodir, 'bin') self.libdir = os.path.join(self.autodir, 'lib') self.testdir = os.path.join(self.autodir, 'tests') self.configdir = os.path.join(self.autodir, 'config') self.site_testdir = os.path.join(self.autodir, 'site_tests') self.profdir = os.path.join(self.autodir, 'profilers') self.tmpdir = os.path.join(self.autodir, 'tmp') self.toolsdir = os.path.join(self.autodir, 'tools') self.resultdir = os.path.join(self.autodir, 'results', options.tag) if not os.path.exists(self.resultdir): os.makedirs(self.resultdir) if not options.cont: self._cleanup_results_dir() logging_manager.configure_logging( client_logging_config.ClientLoggingConfig(), results_dir=self.resultdir, verbose=options.verbose) logging.info('Writing results to %s', self.resultdir) self.drop_caches_between_iterations = False self.drop_caches = drop_caches if self.drop_caches: logging.debug("Dropping caches") utils.drop_caches() self.control = os.path.realpath(control) self._is_continuation = options.cont self.state_file = self.control + '.state' self.current_step_ancestry = [] self.next_step_index = 0 self.testtag = '' self._test_tag_prefix = '' self._load_state() self.pkgmgr = packages.PackageManager( self.autodir, run_function_dargs={'timeout':3600}) self.pkgdir = os.path.join(self.autodir, 'packages') self.run_test_cleanup = self.get_state("__run_test_cleanup", default=True) self.sysinfo = sysinfo.sysinfo(self.resultdir) self._load_sysinfo_state() self.last_boot_tag = self.get_state("__last_boot_tag", default=None) self.tag = self.get_state("__job_tag", default=None) if not options.cont: """ Don't cleanup the tmp dir (which contains the lockfile) in the constructor, this would be a problem for multiple jobs starting at the same time on the same client. Instead do the delete at the server side. We simply create the tmp directory here if it does not already exist. """ if not os.path.exists(self.tmpdir): os.mkdir(self.tmpdir) if not os.path.exists(self.pkgdir): os.mkdir(self.pkgdir) results = os.path.join(self.autodir, 'results') if not os.path.exists(results): os.mkdir(results) download = os.path.join(self.testdir, 'download') if not os.path.exists(download): os.mkdir(download) os.makedirs(os.path.join(self.resultdir, 'analysis')) shutil.copyfile(self.control, os.path.join(self.resultdir, 'control')) self.control = control self.jobtag = options.tag self.log_filename = self.DEFAULT_LOG_FILENAME self.logging = logging_manager.get_logging_manager( manage_stdout_and_stderr=True, redirect_fds=True) self.logging.start_logging() self._init_group_level() self.config = config.config(self) self.harness = harness.select(options.harness, self) self.profilers = profilers.profilers(self) try: tool = self.config_get('boottool.executable') self.bootloader = boottool.boottool(tool) except: pass self.sysinfo.log_per_reboot_data() if not options.cont: self.record('START', None, None) self._increment_group_level() self.harness.run_start() if options.log: self.enable_external_logging() # load the max disk usage rate - default to no monitoring self.max_disk_usage_rate = self.get_state('__monitor_disk', default=0.0) copy_cmdline = set(['console']) if extra_copy_cmdline is not None: copy_cmdline.update(extra_copy_cmdline) # extract console= and other args from cmdline and add them into the # base args that we use for all kernels we install cmdline = utils.read_one_line('/proc/cmdline') kernel_args = [] for karg in cmdline.split(): for param in copy_cmdline: if karg.startswith(param) and \ (len(param) == len(karg) or karg[len(param)] == '='): kernel_args.append(karg) self.config_set('boot.default_args', ' '.join(kernel_args))
def drop_caches_between_iterations(self): if self.job.drop_caches_between_iterations: print "Dropping caches between iterations" utils.drop_caches()
except Exception, info: raise Exception("mount failed: exception = %s, args = %s" % (type(info), info.args)) # If we skipped mkfs we need to wipe the partition clean if skip_mkfs: fs.wipe() # Record the new file system type and options in the disk list disk["mounted"] = True disk["fs_type"] = fs_type disk["fs_mkfs"] = fs_makeopt disk["fs_opts"] = fs_mnt_opt # Try to wipe the file system slate clean utils.drop_caches() # XXX(gps): Remove this code once refactoring is complete to get rid of these # nasty test description strings. def _legacy_str_to_test_flags(fs_desc_string): """Convert a legacy FS_LIST string into a partition.FsOptions instance.""" match = re.search('(.*?)/(.*?)/(.*?)/(.*)$', fs_desc_string.strip()) if not match: raise ValueError('unrecognized FS list entry %r' % fs_desc_string) flags_obj = partition.FsOptions(fstype=match.group(1).strip(), mkfs_flags=match.group(2).strip(), mount_options=match.group(3).strip(), fs_tag=match.group(4).strip()) return flags_obj
def run_once( self, mount_point, file_count, write_size, max_flush_time=1, file_system=None, remove_previous=False, sparse_file=os.path.join(os.getcwd(), "sparse_file"), old_cleanup=False, ): """ Control execution of the test. @param mount_point: the absolute path to the mount point. @param file_count: the number of files to write. @param write_size: the size of each file in MB. @param max_flush_time: the maximum time to wait for the writeback to flush dirty data to disk. Default = 1 minute. @param file_system: the new file system to be mounted, if any. Default = None. @param remove_previous: boolean that allows the removal of previous files before creating a new one. Default = False. @param sparse_file: the absolute path to the sparse file. @param old_cleanup: removes previous mount_point if it exists and is not mounted. Default is False. """ # Check validity of parameters. self._check_parameters(mount_point, write_size, file_count, old_cleanup) # Initialize class variables. self.mount_point = mount_point self.sparse_file = sparse_file self.file_system = file_system # Initialize partition values. self._create_partition() # Flush read and write cache. utils.drop_caches() # Start iterations. logging.info("Starting test operations.") test_start_time = datetime.datetime.now() counter = 1 # Run test until file_count files are successfully written to disk. while counter < file_count: logging.info("Iteration %s.", counter) # Write data to disk. write_completion_time = self._write_data(self.mount_point, counter, write_size) logging.debug("Write time:%s", write_completion_time.strftime("%H:%M:%S")) # Wait until data get synced to disk. time_taken = self._wait_until_data_flushed(write_completion_time, max_flush_time) # Log time statistics. logging.info("Time taken to flush data: %s seconds.", time_taken.seconds) # Check if there is a need to remove the previously written file. if remove_previous: logging.debug("Removing previous file instance.") os.remove(file_name) else: logging.debug("Not removing previous file instance.") # Flush cache. logging.debug("Flush cache between iterations.") utils.drop_caches() # Update the result map. self.result_map[counter] = time_taken.seconds # Increment the counter. counter += 1
def run_once(self, mount_point, file_count, write_size, max_flush_time=1, file_system=None, remove_previous=False, sparse_file=os.path.join(os.getcwd(), 'sparse_file'), old_cleanup=False): """ Control execution of the test. @param mount_point: the absolute path to the mount point. @param file_count: the number of files to write. @param write_size: the size of each file in MB. @param max_flush_time: the maximum time to wait for the writeback to flush dirty data to disk. Default = 1 minute. @param file_system: the new file system to be mounted, if any. Default = None. @param remove_previous: boolean that allows the removal of previous files before creating a new one. Default = False. @param sparse_file: the absolute path to the sparse file. @param old_cleanup: removes previous mount_point if it exists and is not mounted. Default is False. """ # Check validity of parameters. self._check_parameters(mount_point, write_size, file_count, old_cleanup) # Initialize class variables. self.mount_point = mount_point self.sparse_file = sparse_file self.file_system = file_system # Initialize partition values. self._create_partition() # Flush read and write cache. utils.drop_caches() # Start iterations. logging.info('Starting test operations.') test_start_time = datetime.datetime.now() counter = 1 # Run test until file_count files are successfully written to disk. while counter < file_count: logging.info('Iteration %s.', counter) # Write data to disk. write_completion_time = self._write_data(self.mount_point, counter, write_size) logging.debug('Write time:%s', write_completion_time.strftime("%H:%M:%S")) # Wait until data get synced to disk. time_taken = self._wait_until_data_flushed(write_completion_time, max_flush_time) # Log time statistics. logging.info('Time taken to flush data: %s seconds.', time_taken.seconds) # Check if there is a need to remove the previously written file. if remove_previous: logging.debug('Removing previous file instance.') os.remove(file_name) else: logging.debug('Not removing previous file instance.') # Flush cache. logging.debug('Flush cache between iterations.') utils.drop_caches() # Update the result map. self.result_map[counter] = time_taken.seconds # Increment the counter. counter += 1
def drop_caches_between_iterations(self): if self.job.drop_caches_between_iterations: print "Dropping caches between iterations" utils.drop_caches()
def run_fish_test_with_memory_pressure( self, browser, test_url, num_fishes, memory_pressure): """Measure fps under memory pressure. It measure FPS of WebGL aquarium while adding memory pressure. It runs in 2 phases: 1. Allocate non-swappable memory until |memory_to_reserve_mb| is remained. The memory is not accessed after allocated. 2. Run "active" memory consumer in the background. After allocated, Its content is accessed sequentially by page and looped around infinitely. The second phase is opeared in two possible modes: 1. "single" mode, which means only one "active" memory consumer. After running a single memory consumer with a given memory size, it waits for a while to see if system can afford current memory pressure (definition here is FPS > 5). If it does, kill current consumer and launch another consumer with a larger memory size. The process keeps going until system couldn't afford the load. 2. "multiple"mode. It simply launch memory consumers with a given size one by one until system couldn't afford the load (e.g., FPS < 5). In "single" mode, CPU load is lighter so we expect swap in/swap out rate to be correlated to FPS better. In "multiple" mode, since there are multiple busy loop processes, CPU pressure is another significant cause of frame drop. Frame drop can happen easily due to busy CPU instead of memory pressure. @param browser: The Browser object to run the test with. @param test_url: The URL to the aquarium test site. @param num_fishes: The number of fishes to run the test with. @param memory_pressure: Memory pressure parameters. """ consumer_mode = memory_pressure.get('consumer_mode', 'single') memory_to_reserve_mb = memory_pressure.get('memory_to_reserve_mb', 500) # Empirical number to quickly produce memory pressure. if consumer_mode == 'single': default_consumer_size_mb = memory_to_reserve_mb + 100 else: default_consumer_size_mb = memory_to_reserve_mb / 2 consumer_size_mb = memory_pressure.get( 'consumer_size_mb', default_consumer_size_mb) # Setup fish tank. self.setup_webpage(browser, test_url, num_fishes) # Drop all file caches. utils.drop_caches() def fps_near_zero(fps_sampler): """Returns whether recent fps goes down to near 0. @param fps_sampler: A system_sampler.Sampler object. """ last_fps = fps_sampler.get_last_avg_fps(6) if last_fps: logging.info('last fps %f', last_fps) if last_fps <= 5: return True return False max_allocated_mb = 0 # Consume free memory and release them by the end. with memory_eater.consume_free_memory(memory_to_reserve_mb): fps_sampler = system_sampler.SystemSampler( memory_eater.MemoryEater.get_active_consumer_pids) end_condition = functools.partial(fps_near_zero, fps_sampler) with fps_meter.FPSMeter(fps_sampler.sample): # Collects some samples before running memory pressure. time.sleep(5) try: if consumer_mode == 'single': # A single run couldn't generate samples representative # enough. # First runs squeeze more inactive anonymous memory into # zram so in later runs we have a more stable memory # stat. max_allocated_mb = max( memory_eater.run_single_memory_pressure( consumer_size_mb, 100, end_condition, 10, 3, 900), memory_eater.run_single_memory_pressure( consumer_size_mb, 20, end_condition, 10, 3, 900), memory_eater.run_single_memory_pressure( consumer_size_mb, 10, end_condition, 10, 3, 900)) elif consumer_mode == 'multiple': max_allocated_mb = ( memory_eater.run_multi_memory_pressure( consumer_size_mb, end_condition, 10, 900)) else: raise error.TestFail( 'Failed: Unsupported consumer mode.') except memory_eater.TimeoutException as e: raise error.TestFail(e) samples = fps_sampler.get_samples() self.write_samples(samples, 'memory_pressure_fps_samples.txt') self.perf_keyval['num_samples'] = len(samples) self.perf_keyval['max_allocated_mb'] = max_allocated_mb logging.info(self.perf_keyval) self.output_perf_value( description='max_allocated_mb_%d_fishes_reserved_%d_mb' % ( num_fishes, memory_to_reserve_mb), value=max_allocated_mb, units='MB', higher_is_better=True)
def drop_caches_between_iterations(self): if self.job.drop_caches_between_iterations: utils.drop_caches()
except Exception, info: raise Exception("mount failed: exception = %s, args = %s" % (type(info), info.args)) # If we skipped mkfs we need to wipe the partition clean if skip_mkfs: fs.wipe() # Record the new file system type and options in the disk list disk["mounted"] = True disk["fs_type"] = fs_type disk["fs_mkfs"] = fs_makeopt disk["fs_opts"] = fs_mnt_opt # Try to wipe the file system slate clean utils.drop_caches() # XXX(gps): Remove this code once refactoring is complete to get rid of these # nasty test description strings. def _legacy_str_to_test_flags(fs_desc_string): """Convert a legacy FS_LIST string into a partition.FsOptions instance.""" match = re.search('(.*?)/(.*?)/(.*?)/(.*)$', fs_desc_string.strip()) if not match: raise ValueError('unrecognized FS list entry %r' % fs_desc_string) flags_obj = partition.FsOptions(fstype=match.group(1).strip(), mkfs_flags=match.group(2).strip(), mount_options=match.group(3).strip(), fs_tag=match.group(4).strip()) return flags_obj