def execute(args): """Run the bot.""" if not args.skip_install_deps: common.install_dependencies() appengine_path = appengine.find_sdk_path() _setup_bot_directory(args) _setup_environment_and_configs(args, appengine_path) try: original_root_dir = os.path.abspath('.') os.chdir(os.path.join(args.directory, 'clusterfuzz')) if os.getenv('USE_GO_WORKER'): proc = common.execute_async( 'bazel run //go/untrusted_runner:worker', cwd=os.path.join(original_root_dir, 'src')) else: proc = common.execute_async('python src/python/bot/startup/run_bot.py') def _stop_handler(*_): print 'Bot has been stopped. Exit.' proc.kill() signal.signal(signal.SIGTERM, _stop_handler) common.process_proc_output(proc) proc.wait() except KeyboardInterrupt: _stop_handler()
def execute(_): """Run integration tests.""" if sys.version_info.major == 2: print('Skipping integration_tests on Python 2.') return command = 'run_server' indicator = b'Booting worker' try: lines = [] server = common.execute_async( 'python -u butler.py {} --skip-install-deps'.format(command)) test_utils.wait_for_emulator_ready( server, command, indicator, timeout=RUN_SERVER_TIMEOUT, output_lines=lines) # Sleep a small amount of time to ensure the server is definitely ready. time.sleep(1) # Call setup ourselves instead of passing --bootstrap since we have no idea # when that finishes. # TODO(ochang): Make bootstrap a separate butler command and just call that. common.execute( ('python butler.py run setup ' '--non-dry-run --local --config-dir={config_dir}' ).format(config_dir=constants.TEST_CONFIG_DIR), exit_on_error=False) request = urllib.request.urlopen('http://' + constants.DEV_APPSERVER_HOST) request.read() # Raises exception on error except Exception: print('Error occurred:') print(b''.join(lines)) raise finally: server.terminate() # TODO(ochang): Test that bot runs, and do a basic fuzzing session to ensure # things work end to end. print('All end-to-end integration tests passed.')
def execute(args): """Run the bot.""" appengine_path = appengine.find_sdk_path() _setup_bot_directory(args) _setup_environment_and_configs(args, appengine_path) try: os.chdir(os.path.join(args.directory, 'clusterfuzz')) proc = common.execute_async('python src/python/bot/startup/run_bot.py') def _stop_handler(*_): print('Bot has been stopped. Exit.') proc.kill() signal.signal(signal.SIGTERM, _stop_handler) common.process_proc_output(proc) proc.wait() except KeyboardInterrupt: _stop_handler()
def execute(args): """Run the server.""" os.environ['LOCAL_DEVELOPMENT'] = 'True' common.kill_leftover_emulators() if not args.skip_install_deps: common.install_dependencies() # Do this everytime as a past deployment might have changed these. appengine.symlink_dirs() # Deploy all yaml files from test project for basic appengine deployment and # local testing to work. This needs to be called on every iteration as a past # deployment might have overwritten or deleted these config files. yaml_paths = local_config.GAEConfig().get_absolute_path('deployment.prod3') appengine.copy_yamls_and_preprocess(yaml_paths) # Build templates. appengine.build_templates() # Clean storage directory if needed. if args.bootstrap or args.clean: if os.path.exists(args.storage_path): print('Clearing local datastore by removing %s.' % args.storage_path) shutil.rmtree(args.storage_path) if not os.path.exists(args.storage_path): os.makedirs(args.storage_path) # Set up local GCS buckets and symlinks. bootstrap_gcs(args.storage_path) # Start pubsub emulator. pubsub_emulator = test_utils.start_cloud_emulator( 'pubsub', args=['--host-port=' + constants.PUBSUB_EMULATOR_HOST], data_dir=args.storage_path) test_utils.setup_pubsub(constants.TEST_APP_ID) # Start Datastore emulator datastore_emulator = test_utils.start_cloud_emulator( 'datastore', args=['--host-port=' + constants.DATASTORE_EMULATOR_HOST], data_dir=args.storage_path, store_on_disk=True) # Start our custom GCS emulator. local_gcs = common.execute_async('go run emulators/gcs.go -storage-path=' + args.storage_path, cwd='local') if args.bootstrap: bootstrap_db() start_cron_threads() os.environ['APPLICATION_ID'] = constants.TEST_APP_ID os.environ['LOCAL_DEVELOPMENT'] = 'True' os.environ['LOCAL_GCS_BUCKETS_PATH'] = 'local_gcs' os.environ['LOCAL_GCS_SERVER_HOST'] = constants.LOCAL_GCS_SERVER_HOST os.environ['DATASTORE_EMULATOR_HOST'] = constants.DATASTORE_EMULATOR_HOST os.environ['PUBSUB_EMULATOR_HOST'] = constants.PUBSUB_EMULATOR_HOST os.environ['GAE_ENV'] = 'dev' try: cron_server = common.execute_async( 'gunicorn -b :{port} main:app'.format( port=constants.CRON_SERVICE_PORT), cwd=os.path.join('src', 'appengine')) common.execute('gunicorn -b :{port} main:app'.format( port=constants.DEV_APPSERVER_PORT), cwd=os.path.join('src', 'appengine')) except KeyboardInterrupt: print('Server has been stopped. Exit.') cron_server.terminate() datastore_emulator.cleanup() pubsub_emulator.cleanup() local_gcs.terminate()
def execute(args): """Run the server.""" os.environ['LOCAL_DEVELOPMENT'] = 'True' common.kill_leftover_emulators() if not args.skip_install_deps: common.install_dependencies() # Do this everytime as a past deployment might have changed these. appengine.symlink_dirs() # Deploy all yaml files from test project for basic appengine deployment and # local testing to work. This needs to be called on every iteration as a past # deployment might have overwritten or deleted these config files. yaml_paths = local_config.GAEConfig().get_absolute_path('deployment.prod') appengine.copy_yamls_and_preprocess(yaml_paths) # Build templates. appengine.build_templates() # Clean storage directory if needed. if args.bootstrap or args.clean: if os.path.exists(args.storage_path): print 'Clearing local datastore by removing %s.' % args.storage_path shutil.rmtree(args.storage_path) if not os.path.exists(args.storage_path): os.makedirs(args.storage_path) # Set up local GCS buckets and symlinks. bootstrap_gcs(args.storage_path) # Start pubsub emulator. pubsub_emulator = test_utils.start_cloud_emulator( 'pubsub', args=['--host-port=' + constants.PUBSUB_EMULATOR_HOST], data_dir=args.storage_path) test_utils.setup_pubsub(constants.TEST_APP_ID) # Start our custom GCS emulator. local_gcs = common.execute_async( 'bazel run //go/testing/gcs ' '--sandbox_writable_path=$(pwd)/../local/storage/local_gcs ' '-- -storage-path=$(pwd)/../local/storage/local_gcs', cwd='src') if args.bootstrap: bootstrap_db() start_cron_threads() try: common.execute( '{dev_appserver_path} -A {project} --skip_sdk_update_check=1 ' '--storage_path={storage_path} --port={appserver_port} ' '--admin_port={admin_port} ' '--datastore_emulator_port={datastore_emulator_port} ' '--require_indexes=true --log_level={log_level} ' '--dev_appserver_log_level={log_level} ' '--support_datastore_emulator=true ' '--env_var LOCAL_DEVELOPMENT=True ' '--env_var PUBSUB_EMULATOR_HOST={pubsub_emulator_host} ' '--env_var LOCAL_GCS_BUCKETS_PATH=local_gcs ' '--env_var LOCAL_GCS_SERVER_HOST={local_gcs_server_host} ' 'src/appengine src/appengine/cron-service.yaml'.format( dev_appserver_path=_dev_appserver_path(), project=constants.TEST_APP_ID, storage_path=args.storage_path, appserver_port=constants.DEV_APPSERVER_PORT, admin_port=constants.DEV_APPSERVER_ADMIN_PORT, datastore_emulator_port=constants.DATASTORE_EMULATOR_PORT, log_level=args.log_level, pubsub_emulator_host=constants.PUBSUB_EMULATOR_HOST, local_gcs_server_host=constants.LOCAL_GCS_SERVER_HOST)) except KeyboardInterrupt: print 'Server has been stopped. Exit.' pubsub_emulator.cleanup() local_gcs.terminate()