def test_alive(self):
"""Test detection of a job that's not expired."""
# N.B. This test may fail if its run time exceeds more than
# about _MARGIN_SECS seconds.
timestamp = make_timestamp(_TEST_EXPIRATION_AGE, False)
self.assertFalse(
job_directories.is_job_expired(_TEST_EXPIRATION_AGE, timestamp))
def _is_expired(job, age_limit):
"""Return whether job directory is expired for uploading
@param job: _JobDirectory instance.
@param age_limit: Minimum age in days at which a job may be offloaded.
"""
job_timestamp = _cached_get_timestamp_if_finished(job)
if not job_timestamp:
return False
return job_directories.is_job_expired(age_limit, job_timestamp)
def _prune(self, dir_entry, job_complete_time):
"""Prune directory if it is uploaded and expired.
@param dir_entry: Directory entry to offload.
@param job_complete_time: The complete time of the job from the AFE
database.
"""
if not (_is_uploaded(dir_entry)
and job_directories.is_job_expired(self._delete_age,
job_complete_time)):
return
try:
shutil.rmtree(dir_entry)
except OSError as e:
# The wrong file permission can lead call `shutil.rmtree(dir_entry)`
# to raise OSError with message 'Permission denied'. Details can be
# found in crbug.com/536151
_handle_dir_os_error(dir_entry, e.errno==errno.EACCES)
# Try again after the permission issue is fixed.
shutil.rmtree(dir_entry)
def _prune(self, dir_entry, job_complete_time):
"""Prune directory if it is uploaded and expired.
@param dir_entry: Directory entry to offload.
@param job_complete_time: The complete time of the job from the AFE
database.
"""
if not (_is_uploaded(dir_entry) and job_directories.is_job_expired(
self._delete_age, job_complete_time)):
return
try:
shutil.rmtree(dir_entry)
except OSError as e:
# The wrong file permission can lead call
# `shutil.rmtree(dir_entry)` to raise OSError with message
# 'Permission denied'. Details can be found in
# crbug.com/536151
if e.errno == errno.EACCES:
correct_results_folder_permission(dir_entry)
m_permission_error = ('chromeos/autotest/errors/gs_offloader/'
'wrong_permissions_count')
metrics_fields = _get_metrics_fields(dir_entry)
metrics.Counter(m_permission_error).increment(
fields=metrics_fields)
def test_expired(self):
"""Test detection of an expired job."""
timestamp = make_timestamp(_TEST_EXPIRATION_AGE, True)
self.assertTrue(
job_directories.is_job_expired(_TEST_EXPIRATION_AGE, timestamp))
def offload_dir(dir_entry, dest_path, job_complete_time):
"""Offload the specified directory entry to Google storage.
@param dir_entry: Directory entry to offload.
@param dest_path: Location in google storage where we will
offload the directory.
@param job_complete_time: The complete time of the job from the AFE
database.
"""
error = False
stdout_file = None
stderr_file = None
try:
upload_signal_filename = '%s/%s/.GS_UPLOADED' % (
RESULTS_DIR, dir_entry)
if not os.path.isfile(upload_signal_filename):
sanitize_dir(dir_entry)
if DEFAULT_CTS_RESULTS_GSURI:
upload_testresult_files(dir_entry, multiprocessing)
if LIMIT_FILE_COUNT:
limit_file_count(dir_entry)
stdout_file = tempfile.TemporaryFile('w+')
stderr_file = tempfile.TemporaryFile('w+')
process = None
signal.alarm(OFFLOAD_TIMEOUT_SECS)
gs_path = '%s%s' % (gs_uri, dest_path)
process = subprocess.Popen(
get_cmd_list(multiprocessing, dir_entry, gs_path),
stdout=stdout_file, stderr=stderr_file)
process.wait()
signal.alarm(0)
if process.returncode == 0:
dir_size = get_directory_size_kibibytes(dir_entry)
m_offload_count = (
'chromeos/autotest/gs_offloader/jobs_offloaded')
metrics.Counter(m_offload_count).increment()
m_offload_size = ('chromeos/autotest/gs_offloader/'
'kilobytes_transferred')
metrics.Counter(m_offload_size).increment_by(dir_size)
if pubsub_topic:
message = _create_test_result_notification(
gs_path, dir_entry)
msg_ids = pubsub_utils.publish_notifications(
pubsub_topic, [message])
if not msg_ids:
error = True
if not error:
open(upload_signal_filename, 'a').close()
else:
error = True
if os.path.isfile(upload_signal_filename):
if job_directories.is_job_expired(delete_age, job_complete_time):
shutil.rmtree(dir_entry)
except TimeoutException:
m_timeout = 'chromeos/autotest/errors/gs_offloader/timed_out_count'
metrics.Counter(m_timeout).increment()
# If we finished the call to Popen(), we may need to
# terminate the child process. We don't bother calling
# process.poll(); that inherently races because the child
# can die any time it wants.
if process:
try:
process.terminate()
except OSError:
# We don't expect any error other than "No such
# process".
pass
logging.error('Offloading %s timed out after waiting %d '
'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS)
error = True
except OSError as e:
# The wrong file permission can lead call
# `shutil.rmtree(dir_entry)` to raise OSError with message
# 'Permission denied'. Details can be found in
# crbug.com/536151
if e.errno == errno.EACCES:
correct_results_folder_permission(dir_entry)
m_permission_error = ('chromeos/autotest/errors/gs_offloader/'
'wrong_permissions_count')
metrics.Counter(m_permission_error).increment()
finally:
signal.alarm(0)
if error:
# Rewind the log files for stdout and stderr and log
# their contents.
stdout_file.seek(0)
stderr_file.seek(0)
stderr_content = stderr_file.read()
logging.warning('Error occurred when offloading %s:', dir_entry)
logging.warning('Stdout:\n%s \nStderr:\n%s', stdout_file.read(),
stderr_content)
# Some result files may have wrong file permission. Try
# to correct such error so later try can success.
# TODO(dshi): The code is added to correct result files
# with wrong file permission caused by bug 511778. After
# this code is pushed to lab and run for a while to
# clean up these files, following code and function
# correct_results_folder_permission can be deleted.
if 'CommandException: Error opening file' in stderr_content:
correct_results_folder_permission(dir_entry)
if stdout_file:
stdout_file.close()
if stderr_file:
stderr_file.close()