def execute(args):
"""Run the bot."""
if not args.skip_install_deps:
common.install_dependencies()
appengine_path = appengine.find_sdk_path()
_setup_bot_directory(args)
_setup_environment_and_configs(args, appengine_path)
try:
original_root_dir = os.path.abspath('.')
os.chdir(os.path.join(args.directory, 'clusterfuzz'))
if os.getenv('USE_GO_WORKER'):
proc = common.execute_async(
'bazel run //go/untrusted_runner:worker',
cwd=os.path.join(original_root_dir, 'src'))
else:
proc = common.execute_async('python src/python/bot/startup/run_bot.py')
def _stop_handler(*_):
print 'Bot has been stopped. Exit.'
proc.kill()
signal.signal(signal.SIGTERM, _stop_handler)
common.process_proc_output(proc)
proc.wait()
except KeyboardInterrupt:
_stop_handler()
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
elif 'TRAVIS_BRANCH' in os.environ:
_, output = common.execute(
'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
_execute_command_and_track_error('pylint ' + file_path)
_execute_command_and_track_error('yapf -d ' + file_path)
futurize_excludes = ' '.join(
['-x ' + exception for exception in _FUTURIZE_EXCEPTIONS])
futurize_command = 'futurize -0 {excludes} {file_path}'.format(
excludes=futurize_excludes, file_path=file_path)
futurize_output = _execute_command_and_track_error(futurize_command)
if ('No changes to ' not in futurize_output and
'No files need to be modified' not in futurize_output):
# Futurize doesn't modify its return code depending on the result.
_error('Python 3 compatibility error introduced.')
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def _prepare_initial_environment(build_directory):
"""Prepare common environment variables that don't depend on the job."""
# Create a temporary directory to use as ROOT_DIR with a copy of the default
# bot and configuration directories nested under it.
root_dir = environment.get_value('ROOT_DIR')
temp_root_dir = tempfile.mkdtemp()
environment.set_value('ROOT_DIR', temp_root_dir)
common.update_dir(os.path.join(root_dir, 'bot'),
os.path.join(temp_root_dir, 'bot'))
common.update_dir(os.path.join(root_dir, 'configs'),
os.path.join(temp_root_dir, 'configs'))
common.update_dir(os.path.join(root_dir, 'resources'),
os.path.join(temp_root_dir, 'resources'))
common.update_dir(os.path.join(root_dir, 'src'),
os.path.join(temp_root_dir, 'src'))
environment.set_value('CONFIG_DIR_OVERRIDE',
os.path.join(temp_root_dir, 'configs', 'test'))
environment.set_value(
'PYTHONPATH',
os.pathsep.join(
[os.path.join(temp_root_dir, 'src'),
appengine.find_sdk_path()]))
environment.set_bot_environment()
# Overrides that should not be set to the default values.
environment.set_value('APP_DIR', build_directory)
environment.set_value('BUILD_DIR', build_directory)
environment.set_value('BUILDS_DIR', build_directory)
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if "GOOGLE_CLOUDBUILD" in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
py_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.go')
]
for file_path in py_changed_file_paths:
if os.path.exists(file_path):
common.execute('pylint ' + file_path)
common.execute('yapf -d ' + file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if os.path.exists(file_path):
common.execute(golint_path + ' ' + file_path)
_, output = common.execute('gofmt -d ' + file_path)
if output.strip():
sys.exit(1)
def _prepare_initial_environment(build_directory, iterations, verbose):
"""Prepare common environment variables that don't depend on the job."""
# Create a temporary directory to use as ROOT_DIR with a copy of the default
# bot and configuration directories nested under it.
root_dir = environment.get_value('ROOT_DIR')
temp_root_dir = tempfile.mkdtemp()
environment.set_value('ROOT_DIR', temp_root_dir)
def _update_directory(directory_name, ignore_paths=None):
"""Copy a subdirectory from a checkout to a temp directory."""
if not ignore_paths:
ignore_paths = []
shutil.copytree(
os.path.join(root_dir, directory_name),
os.path.join(temp_root_dir, directory_name),
ignore=lambda directory, contents:
contents if directory in ignore_paths else [])
_update_directory('bot')
_update_directory('configs')
_update_directory('resources')
_update_directory(
'src',
ignore_paths=[
os.path.join(root_dir, 'src', 'appengine'),
os.path.join(root_dir, 'src', 'bazel-bin'),
os.path.join(root_dir, 'src', 'bazel-genfiles'),
os.path.join(root_dir, 'src', 'bazel-out'),
os.path.join(root_dir, 'src', 'bazel-src'),
os.path.join(root_dir, 'src', 'python', 'tests'),
])
environment.set_value('CONFIG_DIR_OVERRIDE',
os.path.join(temp_root_dir, 'configs', 'test'))
environment.set_value(
'PYTHONPATH',
os.pathsep.join(
[os.path.join(temp_root_dir, 'src'),
appengine.find_sdk_path()]))
environment.set_bot_environment()
# Overrides that should not be set to the default values.
environment.set_value('APP_DIR', build_directory)
environment.set_value('BUILD_DIR', build_directory)
environment.set_value('BUILDS_DIR', build_directory)
# Some functionality must be disabled when running the tool.
environment.set_value('REPRODUCE_TOOL', True)
environment.set_value('TASK_NAME', 'reproduce')
# Force logging to console for this process and child processes.
if verbose:
environment.set_value('LOG_TO_CONSOLE', True)
if iterations:
environment.set_value('CRASH_RETRIES', iterations)
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv("PYTHONPATH", "")
os.environ["PYTHONPATH"] = appengine.find_sdk_path() + ":" + pythonpath
if "GOOGLE_CLOUDBUILD" in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute("git diff --name-only master FETCH_HEAD")
elif "TRAVIS_BRANCH" in os.environ:
_, output = common.execute(
"git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)")
else:
_, output = common.execute("git diff --name-only FETCH_HEAD")
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths
if f.endswith(".py") and not is_auto_generated_file(f)
]
go_changed_file_paths = [f for f in file_paths if f.endswith(".go")]
yaml_changed_file_paths = [f for f in file_paths if f.endswith(".yaml")]
for file_path in py_changed_file_paths:
_execute_command_and_track_error("pylint " + file_path)
_execute_command_and_track_error("yapf -d " + file_path)
futurize_excludes = " ".join("-x " + exception
for exception in _FUTURIZE_EXCEPTIONS)
futurize_command = "futurize -0 {excludes} {file_path}".format(
excludes=futurize_excludes, file_path=file_path)
futurize_output = _execute_command_and_track_error(futurize_command)
if ("No changes to " not in futurize_output
and "No files need to be modified" not in futurize_output):
# Futurize doesn't modify its return code depending on the result.
_error("Python 3 compatibility error introduced.")
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join("local", "bin", "golint")
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + " " + file_path)
output = _execute_command_and_track_error("gofmt -d " + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print("Linting failed, see errors above.")
sys.exit(1)
else:
print("Linting passed.")
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [
f.decode('utf-8') for f in output.splitlines() if os.path.exists(f)
]
py_changed_file_paths = [
f for f in file_paths
if f.endswith('.py') and not is_auto_generated_file(f)
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
line_length_override = ''
if '_test.py' in file_path:
line_length_override = '--max-line-length=240'
_execute_command_and_track_error(
f'pylint {line_length_override} {file_path}')
_execute_command_and_track_error(f'yapf -d {file_path}')
_execute_command_and_track_error(
f'{formatter.ISORT_CMD} -c {file_path}')
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
elif 'TRAVIS_BRANCH' in os.environ:
_, output = common.execute(
'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
_execute_command_and_track_error('pylint ' + file_path)
_execute_command_and_track_error('yapf -d ' + file_path)
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def _setup():
"""Set up configs and import paths."""
appengine_sdk_path = appengine.find_sdk_path()
sys.path.insert(0, appengine_sdk_path)
# This is necessary for our local environment, where googledatastore (old
# library used only in tests) depends on protobufs which have duplicate
# definitions in google-cloud-datastore. The python implementation of
# protobufs are more relaxed and allow this.
# TODO(ochang): Remove this once we no longer depend on googledatastore.
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
os.environ['ROOT_DIR'] = os.path.abspath('.')
sys.path.insert(0, os.path.abspath(os.path.join('src')))
sys.path.insert(0, os.path.abspath(os.path.join('src', 'third_party')))
# Add our third_party libs before older appengine libs.
from python.base import modules
modules.fix_module_search_paths()
def execute(args):
"""Run the bot."""
appengine_path = appengine.find_sdk_path()
_setup_bot_directory(args)
_setup_environment_and_configs(args, appengine_path)
try:
os.chdir(os.path.join(args.directory, 'clusterfuzz'))
proc = common.execute_async('python src/python/bot/startup/run_bot.py')
def _stop_handler(*_):
print('Bot has been stopped. Exit.')
proc.kill()
signal.signal(signal.SIGTERM, _stop_handler)
common.process_proc_output(proc)
proc.wait()
except KeyboardInterrupt:
_stop_handler()
def execute(args):
"""Run Python unit tests. For unittests involved appengine, sys.path needs
certain modification."""
os.environ['PY_UNITTESTS'] = 'True'
os.environ['CLOUDSDK_PYTHON'] = 'python2'
if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'):
# Set up per-user buckets used by integration tests.
os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET')
os.environ['QUARANTINE_BUCKET'] = common.test_bucket(
'TEST_QUARANTINE_BUCKET')
os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET')
os.environ['COVERAGE_BUCKET'] = common.test_bucket(
'TEST_COVERAGE_BUCKET')
# Kill leftover instances of emulators and dev appserver.
common.kill_leftover_emulators()
# Don't use absolute paths to make it easier to compare results in tests.
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
top_level_dir = os.path.join('src', 'python')
if args.target == 'appengine':
# Build template files.
appengine.build_templates()
test_directory = APPENGINE_TEST_DIRECTORY
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
for i, path in enumerate(sys.path):
if 'third_party' in path:
# Replace third_party with App Engine third_party/.
sys.path[i] = os.path.abspath(
os.path.join('src', 'appengine', 'third_party'))
if sys.version_info.major == 2:
# TODO(ochang): Remove once migrated to Python 3.
appengine_sdk_path = appengine.find_sdk_path()
sys.path.insert(0, appengine_sdk_path)
# Get additional App Engine third party imports.
import dev_appserver
dev_appserver.fix_google_path()
sys.path.extend(dev_appserver.EXTRA_PATHS)
# Loading appengine_main from the current project ensures that any
# changes to configuration there are available to all tests (e.g.
# sys.path modifications, namespaces, etc.)
try:
from src.appengine import main as appengine_main
(appengine_main) # pylint: disable=pointless-statement
except ImportError:
print('Note: unable to import appengine_main.')
# google.auth uses App Engine credentials based on importability of
# google.appengine.api.app_identity.
try:
from google.auth import app_engine as auth_app_engine
if auth_app_engine.app_identity:
auth_app_engine.app_identity = None
except ImportError:
pass
elif args.target == 'core':
test_directory = CORE_TEST_DIRECTORY
else:
# Config module tests.
os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
test_directory = os.path.join(args.config_dir, 'modules')
top_level_dir = None
# Modules may use libs from our App Engine directory.
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
# Fix paths again to get config modules added to the import path.
from python.base import modules
modules.fix_module_search_paths()
# Set expected environment variables.
local_config.ProjectConfig().set_environment()
# Needed for NDB to work with cloud datastore emulator.
os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
if args.verbose:
# Force logging to console for this process and child processes.
os.environ['LOG_TO_CONSOLE'] = 'True'
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if args.pattern is None:
args.pattern = '*_test.py'
if args.parallel:
# TODO(tanin): Support coverage.
run_tests_parallel(args, test_directory, top_level_dir)
else:
run_tests_single_core(args, test_directory, top_level_dir)
def _prepare_initial_environment(build_directory, iterations, verbose):
"""Prepare common environment variables that don't depend on the job."""
# Create a temporary directory to use as ROOT_DIR with a copy of the default
# bot and configuration directories nested under it.
root_dir = environment.get_value("ROOT_DIR")
temp_root_dir = tempfile.mkdtemp()
environment.set_value("ROOT_DIR", temp_root_dir)
def _update_directory(directory_name, ignore_paths=None):
"""Copy a subdirectory from a checkout to a temp directory."""
if not ignore_paths:
ignore_paths = []
shutil.copytree(
os.path.join(root_dir, directory_name),
os.path.join(temp_root_dir, directory_name),
ignore=lambda directory, contents: contents
if directory in ignore_paths else [],
)
_update_directory("bot")
_update_directory("configs")
_update_directory("resources")
_update_directory(
"src",
ignore_paths=[
os.path.join(root_dir, "src", "appengine"),
os.path.join(root_dir, "src", "bazel-bin"),
os.path.join(root_dir, "src", "bazel-genfiles"),
os.path.join(root_dir, "src", "bazel-out"),
os.path.join(root_dir, "src", "bazel-src"),
os.path.join(root_dir, "src", "python", "tests"),
],
)
environment.set_value("CONFIG_DIR_OVERRIDE",
os.path.join(temp_root_dir, "configs", "test"))
environment.set_value(
"PYTHONPATH",
os.pathsep.join(
[os.path.join(temp_root_dir, "src"),
appengine.find_sdk_path()]),
)
environment.set_bot_environment()
# Overrides that should not be set to the default values.
environment.set_value("APP_DIR", build_directory)
environment.set_value("BUILD_DIR", build_directory)
environment.set_value("BUILDS_DIR", build_directory)
# Some functionality must be disabled when running the tool.
environment.set_value("REPRODUCE_TOOL", True)
environment.set_value("TASK_NAME", "reproduce")
# Force logging to console for this process and child processes.
if verbose:
environment.set_value("LOG_TO_CONSOLE", True)
if iterations:
environment.set_value("CRASH_RETRIES", iterations)
