def _emit_offload_metrics(dirpath): """Emit gs offload metrics. @param dirpath: Offloaded directory path. """ dir_size = file_utils.get_directory_size_kibibytes(dirpath) metrics_fields = _get_metrics_fields(dirpath) m_offload_count = ('chromeos/autotest/gs_offloader/jobs_offloaded') metrics.Counter(m_offload_count).increment(fields=metrics_fields) m_offload_size = ('chromeos/autotest/gs_offloader/' 'kilobytes_transferred') metrics.Counter(m_offload_size).increment_by(dir_size, fields=metrics_fields)
def _get_default_size_info(path): """Get the default result size information. In case directory summary is failed to build, assume the test result is not throttled and all result sizes are the size of existing test results. @return: A namedtuple of result size informations, including: client_result_collected_KB: The total size (in KB) of test results collected from test device. Set to be the total size of the given path. original_result_total_KB: The original size (in KB) of test results before being trimmed. Set to be the total size of the given path. result_uploaded_KB: The total size (in KB) of test results to be uploaded. Set to be the total size of the given path. result_throttled: True if test results collection is throttled. It's set to False in this default behavior. """ total_size = file_utils.get_directory_size_kibibytes(path); return result_utils_lib.ResultSizeInfo( client_result_collected_KB=total_size, original_result_total_KB=total_size, result_uploaded_KB=total_size, result_throttled=False)
def _try_offload(self, dir_entry, dest_path, stdout_file, stderr_file): """Offload the specified directory entry to Google storage. @param dir_entry: Directory entry to offload. @param dest_path: Location in google storage where we will offload the directory. @param job_complete_time: The complete time of the job from the AFE database. @param stdout_file: Log file. @param stderr_file: Log file. """ if _is_uploaded(dir_entry): return start_time = time.time() metrics_fields = _get_metrics_fields(dir_entry) es_metadata = _get_es_metadata(dir_entry) error_obj = _OffloadError(start_time, es_metadata) try: sanitize_dir(dir_entry) if DEFAULT_CTS_RESULTS_GSURI: _upload_cts_testresult(dir_entry, self._multiprocessing) if LIMIT_FILE_COUNT: limit_file_count(dir_entry) es_metadata['size_kb'] = file_utils.get_directory_size_kibibytes(dir_entry) process = None with timeout_util.Timeout(OFFLOAD_TIMEOUT_SECS): gs_path = '%s%s' % (self._gs_uri, dest_path) process = subprocess.Popen( _get_cmd_list(self._multiprocessing, dir_entry, gs_path), stdout=stdout_file, stderr=stderr_file) process.wait() _emit_gs_returncode_metric(process.returncode) if process.returncode != 0: raise error_obj _emit_offload_metrics(dir_entry) if self._console_client: gcs_uri = os.path.join(gs_path, os.path.basename(dir_entry)) if not self._console_client.send_test_job_offloaded_message( gcs_uri): raise error_obj _mark_uploaded(dir_entry) except timeout_util.TimeoutError: m_timeout = 'chromeos/autotest/errors/gs_offloader/timed_out_count' metrics.Counter(m_timeout).increment(fields=metrics_fields) # If we finished the call to Popen(), we may need to # terminate the child process. We don't bother calling # process.poll(); that inherently races because the child # can die any time it wants. if process: try: process.terminate() except OSError: # We don't expect any error other than "No such # process". pass logging.error('Offloading %s timed out after waiting %d ' 'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS) raise error_obj