def run():
"""Run update task."""
# Since this code is particularly sensitive for bot stability, continue
# execution but store the exception if anything goes wrong during one of these
# steps.
try:
# Update heartbeat with current time.
data_handler.update_heartbeat()
# Download new layout tests once per day.
update_tests_if_needed()
# Check overall free disk space. If we are running too low, clear all
# data directories like builds, fuzzers, data bundles, etc.
shell.clear_data_directories_on_low_disk_space()
except Exception:
logs.log_error('Error occurred while running update task.')
# Even if there is an exception in one of the other steps, we want to try to
# update the source. If for some reason the source code update fails, it is
# not necessary to run the init scripts.
try:
# If there is a newer revision, exit and let run.py update the source code.
if get_newer_source_revision() is not None:
if environment.is_trusted_host():
from bot.untrusted_runner import host
host.update_worker()
sys.exit(0)
# Run platform specific initialization scripts.
run_platform_init_scripts()
except Exception:
logs.log_error('Error occurred while running update task.')
def track_task_start(task, task_duration):
"""Cache task information."""
persistent_cache.set_value(TASK_PAYLOAD_KEY, task.payload())
persistent_cache.set_value(TASK_END_TIME_KEY, time.time() + task_duration)
# Don't wait on |run_heartbeat|, update task information as soon as it starts.
from datastore import data_handler
data_handler.update_heartbeat(force_update=True)
def track_task_end():
"""Remove cached task information."""
persistent_cache.delete_value(TASK_PAYLOAD_KEY)
persistent_cache.delete_value(TASK_END_TIME_KEY)
# Don't wait on |run_heartbeat|, remove task information as soon as it ends.
from datastore import data_handler
data_handler.update_heartbeat(force_update=True)
def beat(previous_state, log_filename):
"""Run a cycle of heartbeat checks to ensure bot is running."""
# Handle case when run_bot.py script is stuck. If yes, kill its process.
task_end_time = tasks.get_task_end_time()
if psutil and task_end_time and dates.time_has_expired(
task_end_time, seconds=tasks.TASK_COMPLETION_BUFFER):
# Get absolute path to |run_bot| script. We use this to identify unique
# instances of bot running on a particular host.
startup_scripts_directory = environment.get_startup_scripts_directory()
bot_file_path = os.path.join(startup_scripts_directory, 'run_bot')
for process in psutil.process_iter():
try:
command_line = ' '.join(process.cmdline())
except (psutil.AccessDenied, psutil.NoSuchProcess, OSError):
sys.exc_clear()
continue
# Find the process running the main bot script.
if bot_file_path not in command_line:
continue
process_id = process.pid
logs.log('Killing stale bot (pid %d) which seems to have stuck.' %
process_id)
try:
process_handler.terminate_root_and_child_processes(process_id)
except Exception:
logs.log_error('Failed to terminate stale bot processes.')
# Minor cleanup to avoid disk space issues on bot restart.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
shell.clear_testcase_directories()
# Concerned stale processes should be killed. Now, delete the stale task.
tasks.track_task_end()
# Figure out when the log file was last modified.
try:
current_state = str(os.path.getmtime(log_filename))
except Exception:
current_state = None
logs.log('Old state %s, current state %s.' %
(previous_state, current_state))
# Only update the heartbeat if the log file was modified.
if current_state and current_state != previous_state:
# Try updating the heartbeat. If an error occurs, just
# wait and return None.
if not data_handler.update_heartbeat():
return None
# Heartbeat is successfully updated.
return current_state
