def execute(args):
"""Run Python unit tests under v2. For unittests involved appengine, sys.path
needs certain modification."""
os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
local_config.ProjectConfig().set_environment()
if args.local:
os.environ[
'DATASTORE_EMULATOR_HOST'] = constants.DATASTORE_EMULATOR_HOST
os.environ['PUBSUB_EMULATOR_HOST'] = constants.PUBSUB_EMULATOR_HOST
os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
os.environ['LOCAL_DEVELOPMENT'] = 'True'
if not args.non_dry_run:
print(
'Running in dry-run mode, no datastore writes are committed. '
'For permanent modifications, re-run with --non-dry-run.')
script = importlib.import_module('local.butler.scripts.%s' %
args.script_name)
script.execute(args)
if not args.local:
print
print 'Please remember to run the migration individually on all projects.'
print
def _add_env_vars_if_needed(yaml_path, additional_env_vars):
"""Add environment variables to yaml file if necessary."""
# Defer imports since our python paths have to be set up first.
import yaml
from src.python.config import local_config
env_values = local_config.ProjectConfig().get('env')
if additional_env_vars:
env_values.update(additional_env_vars)
if not env_values:
return
with open(yaml_path) as f:
data = yaml.safe_load(f)
if not isinstance(data, dict) or 'service' not in data:
# Not a service.
return
if 'runtime' in data and data['runtime'].startswith('go'):
# Remove HELP_FORMAT as multi-line environment variable values are not
# allowed in Go flex deployment.
env_values.pop('HELP_FORMAT', None)
data.setdefault('env_variables', {}).update(env_values)
with open(yaml_path, 'w') as f:
yaml.safe_dump(data, f)
def _update_alerts(project):
"""Update pubsub topics."""
if not local_config.ProjectConfig().get('monitoring.enabled'):
return
alerts_path = os.path.join(environment.get_config_directory(),
'monitoring', 'alerts.yaml')
with _preprocess_alerts(alerts_path) as processed_alerts_path:
_update_deployment_manager(project, 'alerts', processed_alerts_path)
def _add_env_vars_if_needed(yaml_path):
"""Add environment variables to yaml file if necessary."""
# Defer imports since our python paths have to be set up first.
import yaml
from src.python.config import local_config
env_values = local_config.ProjectConfig().get('env')
if not env_values:
return
with open(yaml_path) as f:
data = yaml.safe_load(f)
if not isinstance(data, dict) or 'service' not in data:
# Not a service.
return
data.setdefault('env_variables', {}).update(env_values)
with open(yaml_path, 'w') as f:
yaml.safe_dump(data, f)
def execute(args):
"""Run Python unit tests. For unittests involved appengine, sys.path needs
certain modification."""
os.environ['PY_UNITTESTS'] = 'True'
os.environ['CLOUDSDK_PYTHON'] = 'python2'
if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'):
# Set up per-user buckets used by integration tests.
os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET')
os.environ['QUARANTINE_BUCKET'] = common.test_bucket(
'TEST_QUARANTINE_BUCKET')
os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET')
os.environ['COVERAGE_BUCKET'] = common.test_bucket(
'TEST_COVERAGE_BUCKET')
# Kill leftover instances of emulators and dev appserver.
common.kill_leftover_emulators()
# Don't use absolute paths to make it easier to compare results in tests.
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
top_level_dir = os.path.join('src', 'python')
if args.target == 'appengine':
# Build template files.
appengine.build_templates()
test_directory = APPENGINE_TEST_DIRECTORY
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
for i, path in enumerate(sys.path):
if 'third_party' in path:
# Replace third_party with App Engine third_party/.
sys.path[i] = os.path.abspath(
os.path.join('src', 'appengine', 'third_party'))
if sys.version_info.major == 2:
# TODO(ochang): Remove once migrated to Python 3.
appengine_sdk_path = appengine.find_sdk_path()
sys.path.insert(0, appengine_sdk_path)
# Get additional App Engine third party imports.
import dev_appserver
dev_appserver.fix_google_path()
sys.path.extend(dev_appserver.EXTRA_PATHS)
# Loading appengine_main from the current project ensures that any
# changes to configuration there are available to all tests (e.g.
# sys.path modifications, namespaces, etc.)
try:
from src.appengine import main as appengine_main
(appengine_main) # pylint: disable=pointless-statement
except ImportError:
print('Note: unable to import appengine_main.')
# google.auth uses App Engine credentials based on importability of
# google.appengine.api.app_identity.
try:
from google.auth import app_engine as auth_app_engine
if auth_app_engine.app_identity:
auth_app_engine.app_identity = None
except ImportError:
pass
elif args.target == 'core':
test_directory = CORE_TEST_DIRECTORY
else:
# Config module tests.
os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
test_directory = os.path.join(args.config_dir, 'modules')
top_level_dir = None
# Modules may use libs from our App Engine directory.
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
# Fix paths again to get config modules added to the import path.
from python.base import modules
modules.fix_module_search_paths()
# Set expected environment variables.
local_config.ProjectConfig().set_environment()
# Needed for NDB to work with cloud datastore emulator.
os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
if args.verbose:
# Force logging to console for this process and child processes.
os.environ['LOG_TO_CONSOLE'] = 'True'
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if args.pattern is None:
args.pattern = '*_test.py'
if args.parallel:
# TODO(tanin): Support coverage.
run_tests_parallel(args, test_directory, top_level_dir)
else:
run_tests_single_core(args, test_directory, top_level_dir)
def _update_alerts(project):
"""Update pubsub topics."""
if local_config.ProjectConfig().get('monitoring.enabled'):
_update_deployment_manager(project, 'alerts',
os.path.join('monitoring', 'alerts.yaml'))
def execute(args):
"""Run Python unit tests. For unittests involved appengine, sys.path needs
certain modification."""
os.environ["PY_UNITTESTS"] = "True"
if os.getenv("INTEGRATION") or os.getenv("UNTRUSTED_RUNNER_TESTS"):
# Set up per-user buckets used by integration tests.
os.environ["CORPUS_BUCKET"] = common.test_bucket("TEST_CORPUS_BUCKET")
os.environ["QUARANTINE_BUCKET"] = common.test_bucket(
"TEST_QUARANTINE_BUCKET")
os.environ["BACKUP_BUCKET"] = common.test_bucket("TEST_BACKUP_BUCKET")
os.environ["COVERAGE_BUCKET"] = common.test_bucket(
"TEST_COVERAGE_BUCKET")
# Kill leftover instances of emulators and dev appserver.
common.kill_leftover_emulators()
# Don't use absolute paths to make it easier to compare results in tests.
os.environ["CONFIG_DIR_OVERRIDE"] = os.path.join(".", "configs", "test")
top_level_dir = os.path.join("src", "python")
if args.target == "appengine":
# Build template files.
appengine.build_templates()
test_directory = APPENGINE_TEST_DIRECTORY
sys.path.insert(0, os.path.abspath(os.path.join("src", "appengine")))
# Get additional App Engine third party imports.
import dev_appserver
sys.path.extend(dev_appserver.EXTRA_PATHS)
# Loading appengine_config from the current project ensures that any
# changes to configuration there are available to all tests (e.g.
# sys.path modifications, namespaces, etc.)
try:
from src.appengine import appengine_config
(appengine_config) # pylint: disable=pointless-statement
except ImportError:
print("Note: unable to import appengine_config.")
elif args.target == "core":
test_directory = CORE_TEST_DIRECTORY
else:
# Config module tests.
os.environ["CONFIG_DIR_OVERRIDE"] = args.config_dir
test_directory = os.path.join(args.config_dir, "modules")
top_level_dir = None
# Modules may use libs from our App Engine directory.
sys.path.insert(0, os.path.abspath(os.path.join("src", "appengine")))
# Fix paths again to get config modules added to the import path.
from python.base import modules
modules.fix_module_search_paths()
# Set expected environment variables.
local_config.ProjectConfig().set_environment()
# Needed for NDB to work with cloud datastore emulator.
os.environ["DATASTORE_USE_PROJECT_ID_AS_APP_ID"] = "true"
if args.verbose:
# Force logging to console for this process and child processes.
os.environ["LOG_TO_CONSOLE"] = "True"
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if args.pattern is None:
args.pattern = "*_test.py"
if args.parallel:
# TODO(tanin): Support coverage.
run_tests_parallel(args, test_directory, top_level_dir)
else:
run_tests_single_core(args, test_directory, top_level_dir)
def execute(args):
"""Run Python unit tests. For unittests involved appengine, sys.path needs
certain modification."""
os.environ['PY_UNITTESTS'] = 'True'
if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'):
# Set up per-user buckets used by integration tests.
os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET')
os.environ['QUARANTINE_BUCKET'] = common.test_bucket(
'TEST_QUARANTINE_BUCKET')
os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET')
os.environ['COVERAGE_BUCKET'] = common.test_bucket('TEST_COVERAGE_BUCKET')
# Kill leftover instances of emulators and dev appserver.
common.kill_leftover_emulators()
# Don't use absolute paths to make it easier to compare results in tests.
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
if args.target == 'appengine':
# Build template files.
appengine.build_templates()
test_directory = APPENGINE_TEST_DIRECTORY
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
# Get additional App Engine third party imports.
import dev_appserver
sys.path.extend(dev_appserver.EXTRA_PATHS)
# Loading appengine_config from the current project ensures that any
# changes to configuration there are available to all tests (e.g.
# sys.path modifications, namespaces, etc.)
try:
from src.appengine import appengine_config
(appengine_config) # pylint: disable=pointless-statement
except ImportError:
print('Note: unable to import appengine_config.')
else:
test_directory = CORE_TEST_DIRECTORY
# Set expected environment variables.
local_config.ProjectConfig().set_environment()
# Needed for NDB to work with cloud datastore emulator.
os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
if args.verbose:
logs.configure_for_tests()
else:
# Disable logging.
logging.disable(logging.CRITICAL)
enable_coverage = args.pattern is None
if args.pattern is None:
args.pattern = '*_test.py'
if args.parallel:
# TODO(tanin): Support coverage.
run_tests_parallel(args, test_directory)
else:
run_tests_single_core(args, test_directory, enable_coverage)
def _update_alerts(project):
"""Update pubsub topics."""
if local_config.ProjectConfig().get("monitoring.enabled"):
_update_deployment_manager(project, "alerts",
os.path.join("monitoring", "alerts.yaml"))
