def _emit_offload_metrics(dirpath):
"""Emit gs offload metrics.
@param dirpath: Offloaded directory path.
"""
dir_size = file_utils.get_directory_size_kibibytes(dirpath)
metrics_fields = _get_metrics_fields(dirpath)
m_offload_count = ('chromeos/autotest/gs_offloader/jobs_offloaded')
metrics.Counter(m_offload_count).increment(fields=metrics_fields)
m_offload_size = ('chromeos/autotest/gs_offloader/'
'kilobytes_transferred')
metrics.Counter(m_offload_size).increment_by(dir_size,
fields=metrics_fields)
def _get_default_size_info(path):
"""Get the default result size information.
In case directory summary is failed to build, assume the test result is not
throttled and all result sizes are the size of existing test results.
@return: A namedtuple of result size informations, including:
client_result_collected_KB: The total size (in KB) of test results
collected from test device. Set to be the total size of the
given path.
original_result_total_KB: The original size (in KB) of test results
before being trimmed. Set to be the total size of the given
path.
result_uploaded_KB: The total size (in KB) of test results to be
uploaded. Set to be the total size of the given path.
result_throttled: True if test results collection is throttled.
It's set to False in this default behavior.
"""
total_size = file_utils.get_directory_size_kibibytes(path);
return result_utils_lib.ResultSizeInfo(
client_result_collected_KB=total_size,
original_result_total_KB=total_size,
result_uploaded_KB=total_size,
result_throttled=False)
def _try_offload(self, dir_entry, dest_path,
stdout_file, stderr_file):
"""Offload the specified directory entry to Google storage.
@param dir_entry: Directory entry to offload.
@param dest_path: Location in google storage where we will
offload the directory.
@param job_complete_time: The complete time of the job from the AFE
database.
@param stdout_file: Log file.
@param stderr_file: Log file.
"""
if _is_uploaded(dir_entry):
return
start_time = time.time()
metrics_fields = _get_metrics_fields(dir_entry)
es_metadata = _get_es_metadata(dir_entry)
error_obj = _OffloadError(start_time, es_metadata)
try:
sanitize_dir(dir_entry)
if DEFAULT_CTS_RESULTS_GSURI:
_upload_cts_testresult(dir_entry, self._multiprocessing)
if LIMIT_FILE_COUNT:
limit_file_count(dir_entry)
es_metadata['size_kb'] = file_utils.get_directory_size_kibibytes(dir_entry)
process = None
with timeout_util.Timeout(OFFLOAD_TIMEOUT_SECS):
gs_path = '%s%s' % (self._gs_uri, dest_path)
process = subprocess.Popen(
_get_cmd_list(self._multiprocessing, dir_entry, gs_path),
stdout=stdout_file, stderr=stderr_file)
process.wait()
_emit_gs_returncode_metric(process.returncode)
if process.returncode != 0:
raise error_obj
_emit_offload_metrics(dir_entry)
if self._console_client:
gcs_uri = os.path.join(gs_path,
os.path.basename(dir_entry))
if not self._console_client.send_test_job_offloaded_message(
gcs_uri):
raise error_obj
_mark_uploaded(dir_entry)
except timeout_util.TimeoutError:
m_timeout = 'chromeos/autotest/errors/gs_offloader/timed_out_count'
metrics.Counter(m_timeout).increment(fields=metrics_fields)
# If we finished the call to Popen(), we may need to
# terminate the child process. We don't bother calling
# process.poll(); that inherently races because the child
# can die any time it wants.
if process:
try:
process.terminate()
except OSError:
# We don't expect any error other than "No such
# process".
pass
logging.error('Offloading %s timed out after waiting %d '
'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS)
raise error_obj