def test_alive(self): """Test detection of a job that's not expired.""" # N.B. This test may fail if its run time exceeds more than # about _MARGIN_SECS seconds. timestamp = make_timestamp(_TEST_EXPIRATION_AGE, False) self.assertFalse( job_directories.is_job_expired(_TEST_EXPIRATION_AGE, timestamp))
def _is_expired(job, age_limit): """Return whether job directory is expired for uploading @param job: _JobDirectory instance. @param age_limit: Minimum age in days at which a job may be offloaded. """ job_timestamp = _cached_get_timestamp_if_finished(job) if not job_timestamp: return False return job_directories.is_job_expired(age_limit, job_timestamp)
def _prune(self, dir_entry, job_complete_time): """Prune directory if it is uploaded and expired. @param dir_entry: Directory entry to offload. @param job_complete_time: The complete time of the job from the AFE database. """ if not (_is_uploaded(dir_entry) and job_directories.is_job_expired(self._delete_age, job_complete_time)): return try: shutil.rmtree(dir_entry) except OSError as e: # The wrong file permission can lead call `shutil.rmtree(dir_entry)` # to raise OSError with message 'Permission denied'. Details can be # found in crbug.com/536151 _handle_dir_os_error(dir_entry, e.errno==errno.EACCES) # Try again after the permission issue is fixed. shutil.rmtree(dir_entry)
def _prune(self, dir_entry, job_complete_time): """Prune directory if it is uploaded and expired. @param dir_entry: Directory entry to offload. @param job_complete_time: The complete time of the job from the AFE database. """ if not (_is_uploaded(dir_entry) and job_directories.is_job_expired( self._delete_age, job_complete_time)): return try: shutil.rmtree(dir_entry) except OSError as e: # The wrong file permission can lead call # `shutil.rmtree(dir_entry)` to raise OSError with message # 'Permission denied'. Details can be found in # crbug.com/536151 if e.errno == errno.EACCES: correct_results_folder_permission(dir_entry) m_permission_error = ('chromeos/autotest/errors/gs_offloader/' 'wrong_permissions_count') metrics_fields = _get_metrics_fields(dir_entry) metrics.Counter(m_permission_error).increment( fields=metrics_fields)
def test_expired(self): """Test detection of an expired job.""" timestamp = make_timestamp(_TEST_EXPIRATION_AGE, True) self.assertTrue( job_directories.is_job_expired(_TEST_EXPIRATION_AGE, timestamp))
def offload_dir(dir_entry, dest_path, job_complete_time): """Offload the specified directory entry to Google storage. @param dir_entry: Directory entry to offload. @param dest_path: Location in google storage where we will offload the directory. @param job_complete_time: The complete time of the job from the AFE database. """ error = False stdout_file = None stderr_file = None try: upload_signal_filename = '%s/%s/.GS_UPLOADED' % ( RESULTS_DIR, dir_entry) if not os.path.isfile(upload_signal_filename): sanitize_dir(dir_entry) if DEFAULT_CTS_RESULTS_GSURI: upload_testresult_files(dir_entry, multiprocessing) if LIMIT_FILE_COUNT: limit_file_count(dir_entry) stdout_file = tempfile.TemporaryFile('w+') stderr_file = tempfile.TemporaryFile('w+') process = None signal.alarm(OFFLOAD_TIMEOUT_SECS) gs_path = '%s%s' % (gs_uri, dest_path) process = subprocess.Popen( get_cmd_list(multiprocessing, dir_entry, gs_path), stdout=stdout_file, stderr=stderr_file) process.wait() signal.alarm(0) if process.returncode == 0: dir_size = get_directory_size_kibibytes(dir_entry) m_offload_count = ( 'chromeos/autotest/gs_offloader/jobs_offloaded') metrics.Counter(m_offload_count).increment() m_offload_size = ('chromeos/autotest/gs_offloader/' 'kilobytes_transferred') metrics.Counter(m_offload_size).increment_by(dir_size) if pubsub_topic: message = _create_test_result_notification( gs_path, dir_entry) msg_ids = pubsub_utils.publish_notifications( pubsub_topic, [message]) if not msg_ids: error = True if not error: open(upload_signal_filename, 'a').close() else: error = True if os.path.isfile(upload_signal_filename): if job_directories.is_job_expired(delete_age, job_complete_time): shutil.rmtree(dir_entry) except TimeoutException: m_timeout = 'chromeos/autotest/errors/gs_offloader/timed_out_count' metrics.Counter(m_timeout).increment() # If we finished the call to Popen(), we may need to # terminate the child process. We don't bother calling # process.poll(); that inherently races because the child # can die any time it wants. if process: try: process.terminate() except OSError: # We don't expect any error other than "No such # process". pass logging.error('Offloading %s timed out after waiting %d ' 'seconds.', dir_entry, OFFLOAD_TIMEOUT_SECS) error = True except OSError as e: # The wrong file permission can lead call # `shutil.rmtree(dir_entry)` to raise OSError with message # 'Permission denied'. Details can be found in # crbug.com/536151 if e.errno == errno.EACCES: correct_results_folder_permission(dir_entry) m_permission_error = ('chromeos/autotest/errors/gs_offloader/' 'wrong_permissions_count') metrics.Counter(m_permission_error).increment() finally: signal.alarm(0) if error: # Rewind the log files for stdout and stderr and log # their contents. stdout_file.seek(0) stderr_file.seek(0) stderr_content = stderr_file.read() logging.warning('Error occurred when offloading %s:', dir_entry) logging.warning('Stdout:\n%s \nStderr:\n%s', stdout_file.read(), stderr_content) # Some result files may have wrong file permission. Try # to correct such error so later try can success. # TODO(dshi): The code is added to correct result files # with wrong file permission caused by bug 511778. After # this code is pushed to lab and run for a while to # clean up these files, following code and function # correct_results_folder_permission can be deleted. if 'CommandException: Error opening file' in stderr_content: correct_results_folder_permission(dir_entry) if stdout_file: stdout_file.close() if stderr_file: stderr_file.close()