def main():
"""Update the heartbeat if there is bot activity."""
if len(sys.argv) < 2:
print('Usage: %s <log file>' % sys.argv[0])
return
environment.set_bot_environment()
logs.configure('run_heartbeat')
log_filename = sys.argv[1]
previous_state = None
# Get absolute path to heartbeat script and interpreter needed to execute it.
startup_scripts_directory = environment.get_startup_scripts_directory()
beat_script_path = os.path.join(startup_scripts_directory, BEAT_SCRIPT)
beat_interpreter = shell.get_interpreter(beat_script_path)
assert beat_interpreter
while True:
beat_command = [
beat_interpreter, beat_script_path,
str(previous_state), log_filename
]
try:
previous_state = subprocess.check_output(beat_command,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logs.log_error('Failed to beat.', output=e.output)
except Exception:
logs.log_error('Failed to beat.')
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
break
def beat(previous_state, log_filename):
"""Run a cycle of heartbeat checks to ensure bot is running."""
# Handle case when run_bot.py script is stuck. If yes, kill its process.
task_end_time = tasks.get_task_end_time()
if psutil and task_end_time and dates.time_has_expired(
task_end_time, seconds=tasks.TASK_COMPLETION_BUFFER):
# Get absolute path to |run_bot| script. We use this to identify unique
# instances of bot running on a particular host.
startup_scripts_directory = environment.get_startup_scripts_directory()
bot_file_path = os.path.join(startup_scripts_directory, 'run_bot')
for process in psutil.process_iter():
try:
command_line = ' '.join(process.cmdline())
except (psutil.AccessDenied, psutil.NoSuchProcess, OSError):
continue
# Find the process running the main bot script.
if bot_file_path not in command_line:
continue
process_id = process.pid
logs.log('Killing stale bot (pid %d) which seems to have stuck.' %
process_id)
try:
process_handler.terminate_root_and_child_processes(process_id)
except Exception:
logs.log_error('Failed to terminate stale bot processes.')
# Minor cleanup to avoid disk space issues on bot restart.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
shell.clear_testcase_directories()
# Concerned stale processes should be killed. Now, delete the stale task.
tasks.track_task_end()
# Figure out when the log file was last modified.
try:
current_state = str(os.path.getmtime(log_filename))
except Exception:
current_state = None
# Only update the heartbeat if the log file was modified.
if current_state and current_state != previous_state:
# Try updating the heartbeat. If an error occurs, just
# wait and return None.
if not data_handler.update_heartbeat():
return None
# Heartbeat is successfully updated.
return current_state
def start_android_heartbeat():
"""Start the android heartbeat (in another process)."""
global _android_heartbeat_handle
if _android_heartbeat_handle:
# If heartbeat is already started, no work to do. Bail out.
return
base_directory = environment.get_startup_scripts_directory()
android_beat_script_path = os.path.join(base_directory,
ANDROID_HEARTBEAT_SCRIPT)
android_beat_interpreter = shell.get_interpreter(android_beat_script_path)
assert android_beat_interpreter
android_beat_command = [android_beat_interpreter, android_beat_script_path]
try:
process_handle = subprocess.Popen(android_beat_command)
except Exception:
logs.log_error('Unable to start android heartbeat process (%s).' %
android_beat_command)
return
# If heartbeat is successfully started, set its handle now.
_android_heartbeat_handle = process_handle
def main():
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print(
'Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
set_start_time()
environment.set_bot_environment()
persistent_cache.initialize()
logs.configure('run')
# Python buffering can otherwise cause exception logs in the child run_*.py
# processes to be lost.
environment.set_value('PYTHONUNBUFFERED', 1)
# Create command strings to launch bot and heartbeat.
base_directory = environment.get_startup_scripts_directory()
log_directory = environment.get_value('LOG_DIR')
bot_log = os.path.join(log_directory, 'bot.log')
bot_script_path = os.path.join(base_directory, BOT_SCRIPT)
bot_interpreter = shell.get_interpreter(bot_script_path)
assert bot_interpreter
bot_command = '%s %s' % (bot_interpreter, bot_script_path)
heartbeat_script_path = os.path.join(base_directory, HEARTBEAT_SCRIPT)
heartbeat_interpreter = shell.get_interpreter(heartbeat_script_path)
assert heartbeat_interpreter
heartbeat_command = '%s %s %s' % (heartbeat_interpreter,
heartbeat_script_path, bot_log)
run_loop(bot_command, heartbeat_command)
logs.log('Exit run.py')