def main():
"""Update the heartbeat if there is bot activity."""
if len(sys.argv) < 2:
print('Usage: %s <log file>' % sys.argv[0])
return
environment.set_bot_environment()
logs.configure('run_heartbeat')
log_filename = sys.argv[1]
previous_state = None
# Get absolute path to heartbeat script and interpreter needed to execute it.
startup_scripts_directory = environment.get_startup_scripts_directory()
beat_script_path = os.path.join(startup_scripts_directory, BEAT_SCRIPT)
beat_interpreter = shell.get_interpreter(beat_script_path)
assert beat_interpreter
while True:
beat_command = [
beat_interpreter, beat_script_path,
str(previous_state), log_filename
]
try:
previous_state = subprocess.check_output(beat_command,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logs.log_error('Failed to beat.', output=e.output)
except Exception:
logs.log_error('Failed to beat.')
# See if our run timed out, if yes bail out.
if data_handler.bot_run_timed_out():
break
def main():
root_directory = environment.get_value('ROOT_DIR')
if not root_directory:
print(
'Please set ROOT_DIR environment variable to the root of the source '
'checkout before running. Exiting.')
print('For an example, check init.bash in the local directory.')
return
set_start_time()
environment.set_bot_environment()
persistent_cache.initialize()
logs.configure('run')
# Python buffering can otherwise cause exception logs in the child run_*.py
# processes to be lost.
environment.set_value('PYTHONUNBUFFERED', 1)
# Create command strings to launch bot and heartbeat.
base_directory = environment.get_startup_scripts_directory()
log_directory = environment.get_value('LOG_DIR')
bot_log = os.path.join(log_directory, 'bot.log')
bot_script_path = os.path.join(base_directory, BOT_SCRIPT)
bot_interpreter = shell.get_interpreter(bot_script_path)
assert bot_interpreter
bot_command = '%s %s' % (bot_interpreter, bot_script_path)
heartbeat_script_path = os.path.join(base_directory, HEARTBEAT_SCRIPT)
heartbeat_interpreter = shell.get_interpreter(heartbeat_script_path)
assert heartbeat_interpreter
heartbeat_command = '%s %s %s' % (heartbeat_interpreter,
heartbeat_script_path, bot_log)
run_loop(bot_command, heartbeat_command)
logs.log('Exit run.py')
def start_android_heartbeat():
"""Start the android heartbeat (in another process)."""
global _android_heartbeat_handle
if _android_heartbeat_handle:
# If heartbeat is already started, no work to do. Bail out.
return
base_directory = environment.get_startup_scripts_directory()
android_beat_script_path = os.path.join(base_directory,
ANDROID_HEARTBEAT_SCRIPT)
android_beat_interpreter = shell.get_interpreter(android_beat_script_path)
assert android_beat_interpreter
android_beat_command = [android_beat_interpreter, android_beat_script_path]
try:
process_handle = subprocess.Popen(android_beat_command)
except Exception:
logs.log_error('Unable to start android heartbeat process (%s).' %
android_beat_command)
return
# If heartbeat is successfully started, set its handle now.
_android_heartbeat_handle = process_handle
def get_command_line_for_application(file_to_run='',
user_profile_index=0,
app_path=None,
app_args=None,
needs_http=False,
write_command_line_file=False,
get_arguments_only=False):
"""Returns the complete command line required to execute application."""
if app_args is None:
app_args = environment.get_value('APP_ARGS')
if app_path is None:
app_path = environment.get_value('APP_PATH')
if not app_path:
# No APP_PATH is available for e.g. grey box fuzzers.
return ''
additional_command_line_flags = get_additional_command_line_flags(
file_to_run)
app_args_append_testcase = environment.get_value(
'APP_ARGS_APPEND_TESTCASE')
app_directory = environment.get_value('APP_DIR')
app_name = environment.get_value('APP_NAME')
apps_argument = environment.get_value('APPS_ARG')
crash_stacks_directory = environment.get_value('CRASH_STACKTRACES_DIR')
debugger = environment.get_value('DEBUGGER_PATH')
device_testcases_directory = android.constants.DEVICE_TESTCASES_DIR
fuzzer_directory = environment.get_value('FUZZER_DIR')
extension_argument = environment.get_value('EXTENSION_ARG')
input_directory = environment.get_value('INPUT_DIR')
launcher = environment.get_value('LAUNCHER_PATH')
is_android = environment.is_android()
root_directory = environment.get_value('ROOT_DIR')
temp_directory = environment.get_value('BOT_TMPDIR')
user_profile_argument = environment.get_value('USER_PROFILE_ARG')
window_argument = environment.get_value('WINDOW_ARG')
user_profile_directory = get_user_profile_directory(user_profile_index)
# Create user profile directory and setup contents if needed.
setup_user_profile_directory_if_needed(user_profile_directory)
# Handle spaces in APP_PATH.
# If application path has spaces, then we need to quote it.
if ' ' in app_path:
app_path = '"%s"' % app_path
interpreter = shell.get_interpreter(app_name)
if get_arguments_only:
# If we are only returning the arguments, do not return the application
# path or anything else required to run it such as an interpreter.
app_path = ''
elif interpreter:
# Prepend command with interpreter if it is a script.
app_path = '%s %s' % (interpreter, app_path)
# Start creating the command line.
command = ''
# Rebase the file_to_run and launcher paths to the worker's root.
if environment.is_trusted_host():
from clusterfuzz._internal.bot.untrusted_runner import file_host
file_to_run = file_host.rebase_to_worker_root(file_to_run)
launcher = file_host.rebase_to_worker_root(launcher)
# Default case.
testcase_path = file_to_run
testcase_filename = os.path.basename(testcase_path)
testcase_directory = os.path.dirname(testcase_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
testcase_http_url = ''
# Determine where |testcase_file_url| should point depending on platform and
# whether or not a launcher script is used.
if file_to_run:
if launcher:
# In the case of launcher scripts, the testcase file to be run resides on
# the host running the launcher script. Thus |testcase_file_url|, which
# may point to a location on the device for Android job types, does not
# apply. Instead, the launcher script should be passed the original file
# to run. By setting |testcase_file_url| to |file_to_run|, we avoid
# duplicating job definitions solely for supporting launcher scripts.
testcase_file_url = file_to_run
# Jobs that have a launcher script which needs to be run on the host will
# have app_name == launcher. In this case don't prepend launcher to
# command - just use app_name.
if os.path.basename(launcher) != app_name:
launcher_with_interpreter = shell.get_execute_command(launcher)
command += launcher_with_interpreter + ' '
elif is_android:
# Android-specific testcase path fixup for fuzzers that don't rely on
# launcher scripts.
local_testcases_directory = environment.get_value('FUZZ_INPUTS')
# Check if the file to run is in fuzzed testcases folder. If yes, then we
# can substitute with a local device path. Otherwise, it is part of some
# data bundle with resource dependencies and we just need to use http
# host forwarder for that.
if file_to_run.startswith(local_testcases_directory):
testcase_relative_path = (
file_to_run[len(local_testcases_directory) + 1:])
testcase_path = os.path.join(device_testcases_directory,
testcase_relative_path)
testcase_file_url = utils.file_path_to_file_url(testcase_path)
else:
# Force use of host_forwarder based on comment above.
needs_http = True
# Check if the testcase needs to be loaded over http.
# TODO(ochang): Make this work for trusted/untrusted.
http_ip = '127.0.0.1'
http_port_1 = environment.get_value('HTTP_PORT_1', 8000)
relative_testcase_path = file_to_run[len(input_directory +
os.path.sep):]
relative_testcase_path = relative_testcase_path.replace('\\', '/')
testcase_http_url = 'http://%s:%d/%s' % (http_ip, http_port_1,
relative_testcase_path)
if needs_http:
# TODO(unassigned): Support https.
testcase_file_url = testcase_http_url
testcase_path = testcase_http_url
# Compose app arguments.
all_app_args = ''
if user_profile_argument:
all_app_args += ' %s=%s' % (user_profile_argument,
user_profile_directory)
if extension_argument and EXTENSIONS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (extension_argument, testcase_directory)
if apps_argument and APPS_PREFIX in testcase_filename:
all_app_args += ' %s=%s' % (apps_argument, testcase_directory)
if window_argument:
all_app_args += ' %s' % window_argument
if additional_command_line_flags:
all_app_args += ' %s' % additional_command_line_flags.strip()
if app_args:
all_app_args += ' %s' % app_args.strip()
# Append %TESTCASE% at end if no testcase pattern is found in app arguments.
if not utils.sub_string_exists_in(
['%TESTCASE%', '%TESTCASE_FILE_URL%', '%TESTCASE_HTTP_URL%'],
all_app_args) and app_args_append_testcase:
all_app_args += ' %TESTCASE%'
all_app_args = all_app_args.strip()
# Build the actual command to run now.
if debugger:
command += '%s ' % debugger
if app_path:
command += app_path
if all_app_args:
command += ' %s' % all_app_args
command = command.replace('%APP_DIR%', app_directory)
command = command.replace('%CRASH_STACKTRACES_DIR%',
crash_stacks_directory)
command = command.replace('%DEVICE_TESTCASES_DIR%',
device_testcases_directory)
command = command.replace('%FUZZER_DIR%', fuzzer_directory)
command = command.replace('%INPUT_DIR%', input_directory)
command = command.replace('%ROOT_DIR%', root_directory)
command = command.replace('%TESTCASE%', testcase_path)
command = command.replace('%TESTCASE_FILE_URL%', testcase_file_url)
command = command.replace('%TESTCASE_HTTP_URL%', testcase_http_url)
command = command.replace('%TMP_DIR%', temp_directory)
command = command.replace('%USER_PROFILE_DIR%', user_profile_directory)
if is_android and not launcher:
# Initial setup phase for command line.
if write_command_line_file:
android.adb.write_command_line_file(command, app_path)
return android.app.get_launch_command(all_app_args, testcase_path,
testcase_file_url)
# Decide which directory we will run the application from.
# We are using |app_directory| since it helps to locate pdbs
# in same directory, other dependencies, etc.
if os.path.exists(app_directory):
os.chdir(app_directory)
return str(command)
def get_non_interpreter_file_test(self):
"""Test that None is returned for a file that doesn't need one. We don't
want empty string since this is easier to than None. """
self.assertIsNone(shell.get_interpreter('executable'))
def get_interpreted_file_test(self):
"""Test correct interpreter is returned for a file that needs one."""
self.assertEqual('python', shell.get_interpreter('run.py'))