def execute(args): """Run the bot.""" if not args.skip_install_deps: common.install_dependencies() appengine_path = appengine.find_sdk_path() _setup_bot_directory(args) _setup_environment_and_configs(args, appengine_path) try: original_root_dir = os.path.abspath('.') os.chdir(os.path.join(args.directory, 'clusterfuzz')) if os.getenv('USE_GO_WORKER'): proc = common.execute_async( 'bazel run //go/untrusted_runner:worker', cwd=os.path.join(original_root_dir, 'src')) else: proc = common.execute_async('python src/python/bot/startup/run_bot.py') def _stop_handler(*_): print 'Bot has been stopped. Exit.' proc.kill() signal.signal(signal.SIGTERM, _stop_handler) common.process_proc_output(proc) proc.wait() except KeyboardInterrupt: _stop_handler()
def execute(_): """Lint changed code.""" pythonpath = os.getenv('PYTHONPATH', '') os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath if 'GOOGLE_CLOUDBUILD' in os.environ: # Explicitly compare against master if we're running on the CI _, output = common.execute('git diff --name-only master FETCH_HEAD') elif 'TRAVIS_BRANCH' in os.environ: _, output = common.execute( 'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)') else: _, output = common.execute('git diff --name-only FETCH_HEAD') file_paths = [f for f in output.splitlines() if os.path.exists(f)] py_changed_file_paths = [ f for f in file_paths if f.endswith('.py') and # Exclude auto-generated files. not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py') ] go_changed_file_paths = [f for f in file_paths if f.endswith('.go')] yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')] for file_path in py_changed_file_paths: _execute_command_and_track_error('pylint ' + file_path) _execute_command_and_track_error('yapf -d ' + file_path) futurize_excludes = ' '.join( ['-x ' + exception for exception in _FUTURIZE_EXCEPTIONS]) futurize_command = 'futurize -0 {excludes} {file_path}'.format( excludes=futurize_excludes, file_path=file_path) futurize_output = _execute_command_and_track_error(futurize_command) if ('No changes to ' not in futurize_output and 'No files need to be modified' not in futurize_output): # Futurize doesn't modify its return code depending on the result. _error('Python 3 compatibility error introduced.') py_import_order(file_path) py_test_init_check(file_path) golint_path = os.path.join('local', 'bin', 'golint') for file_path in go_changed_file_paths: if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS: _execute_command_and_track_error(golint_path + ' ' + file_path) output = _execute_command_and_track_error('gofmt -d ' + file_path) if output.strip(): _error() for file_path in yaml_changed_file_paths: yaml_validate(file_path) for file_path in file_paths: license_validate(file_path) if _error_occurred: print('Linting failed, see errors above.') sys.exit(1) else: print('Linting passed.')
def _prepare_initial_environment(build_directory): """Prepare common environment variables that don't depend on the job.""" # Create a temporary directory to use as ROOT_DIR with a copy of the default # bot and configuration directories nested under it. root_dir = environment.get_value('ROOT_DIR') temp_root_dir = tempfile.mkdtemp() environment.set_value('ROOT_DIR', temp_root_dir) common.update_dir(os.path.join(root_dir, 'bot'), os.path.join(temp_root_dir, 'bot')) common.update_dir(os.path.join(root_dir, 'configs'), os.path.join(temp_root_dir, 'configs')) common.update_dir(os.path.join(root_dir, 'resources'), os.path.join(temp_root_dir, 'resources')) common.update_dir(os.path.join(root_dir, 'src'), os.path.join(temp_root_dir, 'src')) environment.set_value('CONFIG_DIR_OVERRIDE', os.path.join(temp_root_dir, 'configs', 'test')) environment.set_value( 'PYTHONPATH', os.pathsep.join( [os.path.join(temp_root_dir, 'src'), appengine.find_sdk_path()])) environment.set_bot_environment() # Overrides that should not be set to the default values. environment.set_value('APP_DIR', build_directory) environment.set_value('BUILD_DIR', build_directory) environment.set_value('BUILDS_DIR', build_directory)
def execute(_): """Lint changed code.""" pythonpath = os.getenv('PYTHONPATH', '') os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath if "GOOGLE_CLOUDBUILD" in os.environ: # Explicitly compare against master if we're running on the CI _, output = common.execute('git diff --name-only master FETCH_HEAD') else: _, output = common.execute('git diff --name-only FETCH_HEAD') py_changed_file_paths = [ f for f in output.splitlines() if f.endswith('.py') and # Exclude auto-generated files. not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py') ] go_changed_file_paths = [ f for f in output.splitlines() if f.endswith('.go') ] for file_path in py_changed_file_paths: if os.path.exists(file_path): common.execute('pylint ' + file_path) common.execute('yapf -d ' + file_path) golint_path = os.path.join('local', 'bin', 'golint') for file_path in go_changed_file_paths: if os.path.exists(file_path): common.execute(golint_path + ' ' + file_path) _, output = common.execute('gofmt -d ' + file_path) if output.strip(): sys.exit(1)
def _prepare_initial_environment(build_directory, iterations, verbose): """Prepare common environment variables that don't depend on the job.""" # Create a temporary directory to use as ROOT_DIR with a copy of the default # bot and configuration directories nested under it. root_dir = environment.get_value('ROOT_DIR') temp_root_dir = tempfile.mkdtemp() environment.set_value('ROOT_DIR', temp_root_dir) def _update_directory(directory_name, ignore_paths=None): """Copy a subdirectory from a checkout to a temp directory.""" if not ignore_paths: ignore_paths = [] shutil.copytree( os.path.join(root_dir, directory_name), os.path.join(temp_root_dir, directory_name), ignore=lambda directory, contents: contents if directory in ignore_paths else []) _update_directory('bot') _update_directory('configs') _update_directory('resources') _update_directory( 'src', ignore_paths=[ os.path.join(root_dir, 'src', 'appengine'), os.path.join(root_dir, 'src', 'bazel-bin'), os.path.join(root_dir, 'src', 'bazel-genfiles'), os.path.join(root_dir, 'src', 'bazel-out'), os.path.join(root_dir, 'src', 'bazel-src'), os.path.join(root_dir, 'src', 'python', 'tests'), ]) environment.set_value('CONFIG_DIR_OVERRIDE', os.path.join(temp_root_dir, 'configs', 'test')) environment.set_value( 'PYTHONPATH', os.pathsep.join( [os.path.join(temp_root_dir, 'src'), appengine.find_sdk_path()])) environment.set_bot_environment() # Overrides that should not be set to the default values. environment.set_value('APP_DIR', build_directory) environment.set_value('BUILD_DIR', build_directory) environment.set_value('BUILDS_DIR', build_directory) # Some functionality must be disabled when running the tool. environment.set_value('REPRODUCE_TOOL', True) environment.set_value('TASK_NAME', 'reproduce') # Force logging to console for this process and child processes. if verbose: environment.set_value('LOG_TO_CONSOLE', True) if iterations: environment.set_value('CRASH_RETRIES', iterations)
def execute(_): """Lint changed code.""" pythonpath = os.getenv("PYTHONPATH", "") os.environ["PYTHONPATH"] = appengine.find_sdk_path() + ":" + pythonpath if "GOOGLE_CLOUDBUILD" in os.environ: # Explicitly compare against master if we're running on the CI _, output = common.execute("git diff --name-only master FETCH_HEAD") elif "TRAVIS_BRANCH" in os.environ: _, output = common.execute( "git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)") else: _, output = common.execute("git diff --name-only FETCH_HEAD") file_paths = [f for f in output.splitlines() if os.path.exists(f)] py_changed_file_paths = [ f for f in file_paths if f.endswith(".py") and not is_auto_generated_file(f) ] go_changed_file_paths = [f for f in file_paths if f.endswith(".go")] yaml_changed_file_paths = [f for f in file_paths if f.endswith(".yaml")] for file_path in py_changed_file_paths: _execute_command_and_track_error("pylint " + file_path) _execute_command_and_track_error("yapf -d " + file_path) futurize_excludes = " ".join("-x " + exception for exception in _FUTURIZE_EXCEPTIONS) futurize_command = "futurize -0 {excludes} {file_path}".format( excludes=futurize_excludes, file_path=file_path) futurize_output = _execute_command_and_track_error(futurize_command) if ("No changes to " not in futurize_output and "No files need to be modified" not in futurize_output): # Futurize doesn't modify its return code depending on the result. _error("Python 3 compatibility error introduced.") py_import_order(file_path) py_test_init_check(file_path) golint_path = os.path.join("local", "bin", "golint") for file_path in go_changed_file_paths: if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS: _execute_command_and_track_error(golint_path + " " + file_path) output = _execute_command_and_track_error("gofmt -d " + file_path) if output.strip(): _error() for file_path in yaml_changed_file_paths: yaml_validate(file_path) for file_path in file_paths: license_validate(file_path) if _error_occurred: print("Linting failed, see errors above.") sys.exit(1) else: print("Linting passed.")
def execute(_): """Lint changed code.""" pythonpath = os.getenv('PYTHONPATH', '') os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath if 'GOOGLE_CLOUDBUILD' in os.environ: # Explicitly compare against master if we're running on the CI _, output = common.execute('git diff --name-only master FETCH_HEAD') else: _, output = common.execute('git diff --name-only FETCH_HEAD') file_paths = [ f.decode('utf-8') for f in output.splitlines() if os.path.exists(f) ] py_changed_file_paths = [ f for f in file_paths if f.endswith('.py') and not is_auto_generated_file(f) ] go_changed_file_paths = [f for f in file_paths if f.endswith('.go')] yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')] for file_path in py_changed_file_paths: line_length_override = '' if '_test.py' in file_path: line_length_override = '--max-line-length=240' _execute_command_and_track_error( f'pylint {line_length_override} {file_path}') _execute_command_and_track_error(f'yapf -d {file_path}') _execute_command_and_track_error( f'{formatter.ISORT_CMD} -c {file_path}') py_test_init_check(file_path) golint_path = os.path.join('local', 'bin', 'golint') for file_path in go_changed_file_paths: if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS: _execute_command_and_track_error(golint_path + ' ' + file_path) output = _execute_command_and_track_error('gofmt -d ' + file_path) if output.strip(): _error() for file_path in yaml_changed_file_paths: yaml_validate(file_path) for file_path in file_paths: license_validate(file_path) if _error_occurred: print('Linting failed, see errors above.') sys.exit(1) else: print('Linting passed.')
def execute(_): """Lint changed code.""" pythonpath = os.getenv('PYTHONPATH', '') os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath if 'GOOGLE_CLOUDBUILD' in os.environ: # Explicitly compare against master if we're running on the CI _, output = common.execute('git diff --name-only master FETCH_HEAD') elif 'TRAVIS_BRANCH' in os.environ: _, output = common.execute( 'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)') else: _, output = common.execute('git diff --name-only FETCH_HEAD') file_paths = [f for f in output.splitlines() if os.path.exists(f)] py_changed_file_paths = [ f for f in file_paths if f.endswith('.py') and # Exclude auto-generated files. not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py') ] go_changed_file_paths = [f for f in file_paths if f.endswith('.go')] yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')] for file_path in py_changed_file_paths: _execute_command_and_track_error('pylint ' + file_path) _execute_command_and_track_error('yapf -d ' + file_path) py_import_order(file_path) py_test_init_check(file_path) golint_path = os.path.join('local', 'bin', 'golint') for file_path in go_changed_file_paths: if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS: _execute_command_and_track_error(golint_path + ' ' + file_path) output = _execute_command_and_track_error('gofmt -d ' + file_path) if output.strip(): _error() for file_path in yaml_changed_file_paths: yaml_validate(file_path) for file_path in file_paths: license_validate(file_path) if _error_occurred: print('Linting failed, see errors above.') sys.exit(1) else: print('Linting passed.')
def _setup(): """Set up configs and import paths.""" appengine_sdk_path = appengine.find_sdk_path() sys.path.insert(0, appengine_sdk_path) # This is necessary for our local environment, where googledatastore (old # library used only in tests) depends on protobufs which have duplicate # definitions in google-cloud-datastore. The python implementation of # protobufs are more relaxed and allow this. # TODO(ochang): Remove this once we no longer depend on googledatastore. os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python' os.environ['ROOT_DIR'] = os.path.abspath('.') sys.path.insert(0, os.path.abspath(os.path.join('src'))) sys.path.insert(0, os.path.abspath(os.path.join('src', 'third_party'))) # Add our third_party libs before older appengine libs. from python.base import modules modules.fix_module_search_paths()
def execute(args): """Run the bot.""" appengine_path = appengine.find_sdk_path() _setup_bot_directory(args) _setup_environment_and_configs(args, appengine_path) try: os.chdir(os.path.join(args.directory, 'clusterfuzz')) proc = common.execute_async('python src/python/bot/startup/run_bot.py') def _stop_handler(*_): print('Bot has been stopped. Exit.') proc.kill() signal.signal(signal.SIGTERM, _stop_handler) common.process_proc_output(proc) proc.wait() except KeyboardInterrupt: _stop_handler()
def execute(args): """Run Python unit tests. For unittests involved appengine, sys.path needs certain modification.""" os.environ['PY_UNITTESTS'] = 'True' os.environ['CLOUDSDK_PYTHON'] = 'python2' if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'): # Set up per-user buckets used by integration tests. os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET') os.environ['QUARANTINE_BUCKET'] = common.test_bucket( 'TEST_QUARANTINE_BUCKET') os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET') os.environ['COVERAGE_BUCKET'] = common.test_bucket( 'TEST_COVERAGE_BUCKET') # Kill leftover instances of emulators and dev appserver. common.kill_leftover_emulators() # Don't use absolute paths to make it easier to compare results in tests. os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test') top_level_dir = os.path.join('src', 'python') if args.target == 'appengine': # Build template files. appengine.build_templates() test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) for i, path in enumerate(sys.path): if 'third_party' in path: # Replace third_party with App Engine third_party/. sys.path[i] = os.path.abspath( os.path.join('src', 'appengine', 'third_party')) if sys.version_info.major == 2: # TODO(ochang): Remove once migrated to Python 3. appengine_sdk_path = appengine.find_sdk_path() sys.path.insert(0, appengine_sdk_path) # Get additional App Engine third party imports. import dev_appserver dev_appserver.fix_google_path() sys.path.extend(dev_appserver.EXTRA_PATHS) # Loading appengine_main from the current project ensures that any # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: from src.appengine import main as appengine_main (appengine_main) # pylint: disable=pointless-statement except ImportError: print('Note: unable to import appengine_main.') # google.auth uses App Engine credentials based on importability of # google.appengine.api.app_identity. try: from google.auth import app_engine as auth_app_engine if auth_app_engine.app_identity: auth_app_engine.app_identity = None except ImportError: pass elif args.target == 'core': test_directory = CORE_TEST_DIRECTORY else: # Config module tests. os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir test_directory = os.path.join(args.config_dir, 'modules') top_level_dir = None # Modules may use libs from our App Engine directory. sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) # Fix paths again to get config modules added to the import path. from python.base import modules modules.fix_module_search_paths() # Set expected environment variables. local_config.ProjectConfig().set_environment() # Needed for NDB to work with cloud datastore emulator. os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true' if args.verbose: # Force logging to console for this process and child processes. os.environ['LOG_TO_CONSOLE'] = 'True' else: # Disable logging. logging.disable(logging.CRITICAL) if args.pattern is None: args.pattern = '*_test.py' if args.parallel: # TODO(tanin): Support coverage. run_tests_parallel(args, test_directory, top_level_dir) else: run_tests_single_core(args, test_directory, top_level_dir)
def _prepare_initial_environment(build_directory, iterations, verbose): """Prepare common environment variables that don't depend on the job.""" # Create a temporary directory to use as ROOT_DIR with a copy of the default # bot and configuration directories nested under it. root_dir = environment.get_value("ROOT_DIR") temp_root_dir = tempfile.mkdtemp() environment.set_value("ROOT_DIR", temp_root_dir) def _update_directory(directory_name, ignore_paths=None): """Copy a subdirectory from a checkout to a temp directory.""" if not ignore_paths: ignore_paths = [] shutil.copytree( os.path.join(root_dir, directory_name), os.path.join(temp_root_dir, directory_name), ignore=lambda directory, contents: contents if directory in ignore_paths else [], ) _update_directory("bot") _update_directory("configs") _update_directory("resources") _update_directory( "src", ignore_paths=[ os.path.join(root_dir, "src", "appengine"), os.path.join(root_dir, "src", "bazel-bin"), os.path.join(root_dir, "src", "bazel-genfiles"), os.path.join(root_dir, "src", "bazel-out"), os.path.join(root_dir, "src", "bazel-src"), os.path.join(root_dir, "src", "python", "tests"), ], ) environment.set_value("CONFIG_DIR_OVERRIDE", os.path.join(temp_root_dir, "configs", "test")) environment.set_value( "PYTHONPATH", os.pathsep.join( [os.path.join(temp_root_dir, "src"), appengine.find_sdk_path()]), ) environment.set_bot_environment() # Overrides that should not be set to the default values. environment.set_value("APP_DIR", build_directory) environment.set_value("BUILD_DIR", build_directory) environment.set_value("BUILDS_DIR", build_directory) # Some functionality must be disabled when running the tool. environment.set_value("REPRODUCE_TOOL", True) environment.set_value("TASK_NAME", "reproduce") # Force logging to console for this process and child processes. if verbose: environment.set_value("LOG_TO_CONSOLE", True) if iterations: environment.set_value("CRASH_RETRIES", iterations)