def main(): """Update the heartbeat if there is bot activity.""" if len(sys.argv) < 2: print('Usage: %s <log file>' % sys.argv[0]) return environment.set_bot_environment() logs.configure('run_heartbeat') log_filename = sys.argv[1] previous_state = None # Get absolute path to heartbeat script and interpreter needed to execute it. startup_scripts_directory = environment.get_startup_scripts_directory() beat_script_path = os.path.join(startup_scripts_directory, BEAT_SCRIPT) beat_interpreter = shell.get_interpreter(beat_script_path) assert beat_interpreter while True: beat_command = [ beat_interpreter, beat_script_path, str(previous_state), log_filename ] try: previous_state = subprocess.check_output( beat_command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: logs.log_error('Failed to beat.', output=e.output) except Exception: logs.log_error('Failed to beat.') # See if our run timed out, if yes bail out. if data_handler.bot_run_timed_out(): break
def main(): root_directory = environment.get_value('ROOT_DIR') if not root_directory: print('Please set ROOT_DIR environment variable to the root of the source ' 'checkout before running. Exiting.') print('For an example, check init.bash in the local directory.') return environment.set_bot_environment() persistent_cache.initialize() logs.configure('run') # Create command strings to launch bot and heartbeat. base_directory = environment.get_startup_scripts_directory() log_directory = environment.get_value('LOG_DIR') bot_log = os.path.join(log_directory, 'bot.log') bot_script_path = os.path.join(base_directory, BOT_SCRIPT) bot_interpreter = shell.get_interpreter(bot_script_path) assert bot_interpreter bot_command = '%s %s' % (bot_interpreter, bot_script_path) heartbeat_script_path = os.path.join(base_directory, HEARTBEAT_SCRIPT) heartbeat_interpreter = shell.get_interpreter(heartbeat_script_path) assert heartbeat_interpreter heartbeat_command = '%s %s %s' % (heartbeat_interpreter, heartbeat_script_path, bot_log) run_loop(bot_command, heartbeat_command) logs.log('Exit run.py')
def beat(previous_state, log_filename): """Run a cycle of heartbeat checks to ensure bot is running.""" # Handle case when run_bot.py script is stuck. If yes, kill its process. task_end_time = tasks.get_task_end_time() if psutil and task_end_time and dates.time_has_expired( task_end_time, seconds=tasks.TASK_COMPLETION_BUFFER): # Get absolute path to |run_bot| script. We use this to identify unique # instances of bot running on a particular host. startup_scripts_directory = environment.get_startup_scripts_directory() bot_file_path = os.path.join(startup_scripts_directory, 'run_bot') for process in psutil.process_iter(): try: command_line = ' '.join(process.cmdline()) except (psutil.AccessDenied, psutil.NoSuchProcess, OSError): sys.exc_clear() continue # Find the process running the main bot script. if bot_file_path not in command_line: continue process_id = process.pid logs.log('Killing stale bot (pid %d) which seems to have stuck.' % process_id) try: process_handler.terminate_root_and_child_processes(process_id) except Exception: logs.log_error('Failed to terminate stale bot processes.') # Minor cleanup to avoid disk space issues on bot restart. process_handler.terminate_stale_application_instances() shell.clear_temp_directory() shell.clear_testcase_directories() # Concerned stale processes should be killed. Now, delete the stale task. tasks.track_task_end() # Figure out when the log file was last modified. try: current_state = str(os.path.getmtime(log_filename)) except Exception: current_state = None logs.log('Old state %s, current state %s.' % (previous_state, current_state)) # Only update the heartbeat if the log file was modified. if current_state and current_state != previous_state: # Try updating the heartbeat. If an error occurs, just # wait and return None. if not data_handler.update_heartbeat(): return None # Heartbeat is successfully updated. return current_state