def update_heartbeat(force_update=False):
"""Updates heartbeat with current timestamp and log data."""
# Check if the heartbeat was recently updated. If yes, bail out.
last_modified_time = persistent_cache.get_value(
HEARTBEAT_LAST_UPDATE_KEY, constructor=datetime.datetime.utcfromtimestamp)
if (not force_update and last_modified_time and not dates.time_has_expired(
last_modified_time, seconds=data_types.HEARTBEAT_WAIT_INTERVAL)):
return 0
bot_name = environment.get_value('BOT_NAME')
current_time = datetime.datetime.utcnow()
try:
heartbeat = ndb.Key(data_types.Heartbeat, bot_name).get()
if not heartbeat:
heartbeat = data_types.Heartbeat()
heartbeat.bot_name = bot_name
heartbeat.key = ndb.Key(data_types.Heartbeat, bot_name)
heartbeat.task_payload = tasks.get_task_payload()
heartbeat.task_end_time = tasks.get_task_end_time()
heartbeat.last_beat_time = current_time
heartbeat.source_version = utils.current_source_version()
heartbeat.put()
persistent_cache.set_value(
HEARTBEAT_LAST_UPDATE_KEY, time.time(), persist_across_reboots=True)
except:
logs.log_error('Unable to update heartbeat.')
return 0
return 1
def beat(previous_state, log_filename):
"""Run a cycle of heartbeat checks to ensure bot is running."""
# Handle case when run_bot.py script is stuck. If yes, kill its process.
task_end_time = tasks.get_task_end_time()
if psutil and task_end_time and dates.time_has_expired(
task_end_time, seconds=tasks.TASK_COMPLETION_BUFFER):
# Get absolute path to |run_bot| script. We use this to identify unique
# instances of bot running on a particular host.
startup_scripts_directory = environment.get_startup_scripts_directory()
bot_file_path = os.path.join(startup_scripts_directory, 'run_bot')
for process in psutil.process_iter():
try:
command_line = ' '.join(process.cmdline())
except (psutil.AccessDenied, psutil.NoSuchProcess, OSError):
sys.exc_clear()
continue
# Find the process running the main bot script.
if bot_file_path not in command_line:
continue
process_id = process.pid
logs.log('Killing stale bot (pid %d) which seems to have stuck.' %
process_id)
try:
process_handler.terminate_root_and_child_processes(process_id)
except Exception:
logs.log_error('Failed to terminate stale bot processes.')
# Minor cleanup to avoid disk space issues on bot restart.
process_handler.terminate_stale_application_instances()
shell.clear_temp_directory()
shell.clear_testcase_directories()
# Concerned stale processes should be killed. Now, delete the stale task.
tasks.track_task_end()
# Figure out when the log file was last modified.
try:
current_state = str(os.path.getmtime(log_filename))
except Exception:
current_state = None
logs.log('Old state %s, current state %s.' %
(previous_state, current_state))
# Only update the heartbeat if the log file was modified.
if current_state and current_state != previous_state:
# Try updating the heartbeat. If an error occurs, just
# wait and return None.
if not data_handler.update_heartbeat():
return None
# Heartbeat is successfully updated.
return current_state