def execute(args):
"""Run the bot."""
if not args.skip_install_deps:
common.install_dependencies()
appengine_path = appengine.find_sdk_path()
_setup_bot_directory(args)
_setup_environment_and_configs(args, appengine_path)
try:
original_root_dir = os.path.abspath('.')
os.chdir(os.path.join(args.directory, 'clusterfuzz'))
if os.getenv('USE_GO_WORKER'):
proc = common.execute_async(
'bazel run //go/untrusted_runner:worker',
cwd=os.path.join(original_root_dir, 'src'))
else:
proc = common.execute_async('python src/python/bot/startup/run_bot.py')
def _stop_handler(*_):
print 'Bot has been stopped. Exit.'
proc.kill()
signal.signal(signal.SIGTERM, _stop_handler)
common.process_proc_output(proc)
proc.wait()
except KeyboardInterrupt:
_stop_handler()
def execute(_):
"""Run integration tests."""
if sys.version_info.major == 2:
print('Skipping integration_tests on Python 2.')
return
command = 'run_server'
indicator = b'Booting worker'
try:
lines = []
server = common.execute_async(
'python -u butler.py {} --skip-install-deps'.format(command))
test_utils.wait_for_emulator_ready(
server,
command,
indicator,
timeout=RUN_SERVER_TIMEOUT,
output_lines=lines)
# Sleep a small amount of time to ensure the server is definitely ready.
time.sleep(1)
# Call setup ourselves instead of passing --bootstrap since we have no idea
# when that finishes.
# TODO(ochang): Make bootstrap a separate butler command and just call that.
common.execute(
('python butler.py run setup '
'--non-dry-run --local --config-dir={config_dir}'
).format(config_dir=constants.TEST_CONFIG_DIR),
exit_on_error=False)
request = urllib.request.urlopen('http://' + constants.DEV_APPSERVER_HOST)
request.read() # Raises exception on error
except Exception:
print('Error occurred:')
print(b''.join(lines))
raise
finally:
server.terminate()
# TODO(ochang): Test that bot runs, and do a basic fuzzing session to ensure
# things work end to end.
print('All end-to-end integration tests passed.')
def execute(args):
"""Run the bot."""
appengine_path = appengine.find_sdk_path()
_setup_bot_directory(args)
_setup_environment_and_configs(args, appengine_path)
try:
os.chdir(os.path.join(args.directory, 'clusterfuzz'))
proc = common.execute_async('python src/python/bot/startup/run_bot.py')
def _stop_handler(*_):
print('Bot has been stopped. Exit.')
proc.kill()
signal.signal(signal.SIGTERM, _stop_handler)
common.process_proc_output(proc)
proc.wait()
except KeyboardInterrupt:
_stop_handler()
def execute(args):
"""Run the server."""
os.environ['LOCAL_DEVELOPMENT'] = 'True'
common.kill_leftover_emulators()
if not args.skip_install_deps:
common.install_dependencies()
# Do this everytime as a past deployment might have changed these.
appengine.symlink_dirs()
# Deploy all yaml files from test project for basic appengine deployment and
# local testing to work. This needs to be called on every iteration as a past
# deployment might have overwritten or deleted these config files.
yaml_paths = local_config.GAEConfig().get_absolute_path('deployment.prod3')
appengine.copy_yamls_and_preprocess(yaml_paths)
# Build templates.
appengine.build_templates()
# Clean storage directory if needed.
if args.bootstrap or args.clean:
if os.path.exists(args.storage_path):
print('Clearing local datastore by removing %s.' %
args.storage_path)
shutil.rmtree(args.storage_path)
if not os.path.exists(args.storage_path):
os.makedirs(args.storage_path)
# Set up local GCS buckets and symlinks.
bootstrap_gcs(args.storage_path)
# Start pubsub emulator.
pubsub_emulator = test_utils.start_cloud_emulator(
'pubsub',
args=['--host-port=' + constants.PUBSUB_EMULATOR_HOST],
data_dir=args.storage_path)
test_utils.setup_pubsub(constants.TEST_APP_ID)
# Start Datastore emulator
datastore_emulator = test_utils.start_cloud_emulator(
'datastore',
args=['--host-port=' + constants.DATASTORE_EMULATOR_HOST],
data_dir=args.storage_path,
store_on_disk=True)
# Start our custom GCS emulator.
local_gcs = common.execute_async('go run emulators/gcs.go -storage-path=' +
args.storage_path,
cwd='local')
if args.bootstrap:
bootstrap_db()
start_cron_threads()
os.environ['APPLICATION_ID'] = constants.TEST_APP_ID
os.environ['LOCAL_DEVELOPMENT'] = 'True'
os.environ['LOCAL_GCS_BUCKETS_PATH'] = 'local_gcs'
os.environ['LOCAL_GCS_SERVER_HOST'] = constants.LOCAL_GCS_SERVER_HOST
os.environ['DATASTORE_EMULATOR_HOST'] = constants.DATASTORE_EMULATOR_HOST
os.environ['PUBSUB_EMULATOR_HOST'] = constants.PUBSUB_EMULATOR_HOST
os.environ['GAE_ENV'] = 'dev'
try:
cron_server = common.execute_async(
'gunicorn -b :{port} main:app'.format(
port=constants.CRON_SERVICE_PORT),
cwd=os.path.join('src', 'appengine'))
common.execute('gunicorn -b :{port} main:app'.format(
port=constants.DEV_APPSERVER_PORT),
cwd=os.path.join('src', 'appengine'))
except KeyboardInterrupt:
print('Server has been stopped. Exit.')
cron_server.terminate()
datastore_emulator.cleanup()
pubsub_emulator.cleanup()
local_gcs.terminate()
def execute(args):
"""Run the server."""
os.environ['LOCAL_DEVELOPMENT'] = 'True'
common.kill_leftover_emulators()
if not args.skip_install_deps:
common.install_dependencies()
# Do this everytime as a past deployment might have changed these.
appengine.symlink_dirs()
# Deploy all yaml files from test project for basic appengine deployment and
# local testing to work. This needs to be called on every iteration as a past
# deployment might have overwritten or deleted these config files.
yaml_paths = local_config.GAEConfig().get_absolute_path('deployment.prod')
appengine.copy_yamls_and_preprocess(yaml_paths)
# Build templates.
appengine.build_templates()
# Clean storage directory if needed.
if args.bootstrap or args.clean:
if os.path.exists(args.storage_path):
print 'Clearing local datastore by removing %s.' % args.storage_path
shutil.rmtree(args.storage_path)
if not os.path.exists(args.storage_path):
os.makedirs(args.storage_path)
# Set up local GCS buckets and symlinks.
bootstrap_gcs(args.storage_path)
# Start pubsub emulator.
pubsub_emulator = test_utils.start_cloud_emulator(
'pubsub',
args=['--host-port=' + constants.PUBSUB_EMULATOR_HOST],
data_dir=args.storage_path)
test_utils.setup_pubsub(constants.TEST_APP_ID)
# Start our custom GCS emulator.
local_gcs = common.execute_async(
'bazel run //go/testing/gcs '
'--sandbox_writable_path=$(pwd)/../local/storage/local_gcs '
'-- -storage-path=$(pwd)/../local/storage/local_gcs',
cwd='src')
if args.bootstrap:
bootstrap_db()
start_cron_threads()
try:
common.execute(
'{dev_appserver_path} -A {project} --skip_sdk_update_check=1 '
'--storage_path={storage_path} --port={appserver_port} '
'--admin_port={admin_port} '
'--datastore_emulator_port={datastore_emulator_port} '
'--require_indexes=true --log_level={log_level} '
'--dev_appserver_log_level={log_level} '
'--support_datastore_emulator=true '
'--env_var LOCAL_DEVELOPMENT=True '
'--env_var PUBSUB_EMULATOR_HOST={pubsub_emulator_host} '
'--env_var LOCAL_GCS_BUCKETS_PATH=local_gcs '
'--env_var LOCAL_GCS_SERVER_HOST={local_gcs_server_host} '
'src/appengine src/appengine/cron-service.yaml'.format(
dev_appserver_path=_dev_appserver_path(),
project=constants.TEST_APP_ID,
storage_path=args.storage_path,
appserver_port=constants.DEV_APPSERVER_PORT,
admin_port=constants.DEV_APPSERVER_ADMIN_PORT,
datastore_emulator_port=constants.DATASTORE_EMULATOR_PORT,
log_level=args.log_level,
pubsub_emulator_host=constants.PUBSUB_EMULATOR_HOST,
local_gcs_server_host=constants.LOCAL_GCS_SERVER_HOST))
except KeyboardInterrupt:
print 'Server has been stopped. Exit.'
pubsub_emulator.cleanup()
local_gcs.terminate()
