def execute(args): """Run Python unit tests under v2. For unittests involved appengine, sys.path needs certain modification.""" os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir local_config.ProjectConfig().set_environment() if args.local: os.environ[ 'DATASTORE_EMULATOR_HOST'] = constants.DATASTORE_EMULATOR_HOST os.environ['PUBSUB_EMULATOR_HOST'] = constants.PUBSUB_EMULATOR_HOST os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true' os.environ['LOCAL_DEVELOPMENT'] = 'True' if not args.non_dry_run: print( 'Running in dry-run mode, no datastore writes are committed. ' 'For permanent modifications, re-run with --non-dry-run.') script = importlib.import_module('local.butler.scripts.%s' % args.script_name) script.execute(args) if not args.local: print print 'Please remember to run the migration individually on all projects.' print
def _add_env_vars_if_needed(yaml_path, additional_env_vars): """Add environment variables to yaml file if necessary.""" # Defer imports since our python paths have to be set up first. import yaml from src.python.config import local_config env_values = local_config.ProjectConfig().get('env') if additional_env_vars: env_values.update(additional_env_vars) if not env_values: return with open(yaml_path) as f: data = yaml.safe_load(f) if not isinstance(data, dict) or 'service' not in data: # Not a service. return if 'runtime' in data and data['runtime'].startswith('go'): # Remove HELP_FORMAT as multi-line environment variable values are not # allowed in Go flex deployment. env_values.pop('HELP_FORMAT', None) data.setdefault('env_variables', {}).update(env_values) with open(yaml_path, 'w') as f: yaml.safe_dump(data, f)
def _update_alerts(project): """Update pubsub topics.""" if not local_config.ProjectConfig().get('monitoring.enabled'): return alerts_path = os.path.join(environment.get_config_directory(), 'monitoring', 'alerts.yaml') with _preprocess_alerts(alerts_path) as processed_alerts_path: _update_deployment_manager(project, 'alerts', processed_alerts_path)
def _add_env_vars_if_needed(yaml_path): """Add environment variables to yaml file if necessary.""" # Defer imports since our python paths have to be set up first. import yaml from src.python.config import local_config env_values = local_config.ProjectConfig().get('env') if not env_values: return with open(yaml_path) as f: data = yaml.safe_load(f) if not isinstance(data, dict) or 'service' not in data: # Not a service. return data.setdefault('env_variables', {}).update(env_values) with open(yaml_path, 'w') as f: yaml.safe_dump(data, f)
def execute(args): """Run Python unit tests. For unittests involved appengine, sys.path needs certain modification.""" os.environ['PY_UNITTESTS'] = 'True' os.environ['CLOUDSDK_PYTHON'] = 'python2' if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'): # Set up per-user buckets used by integration tests. os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET') os.environ['QUARANTINE_BUCKET'] = common.test_bucket( 'TEST_QUARANTINE_BUCKET') os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET') os.environ['COVERAGE_BUCKET'] = common.test_bucket( 'TEST_COVERAGE_BUCKET') # Kill leftover instances of emulators and dev appserver. common.kill_leftover_emulators() # Don't use absolute paths to make it easier to compare results in tests. os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test') top_level_dir = os.path.join('src', 'python') if args.target == 'appengine': # Build template files. appengine.build_templates() test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) for i, path in enumerate(sys.path): if 'third_party' in path: # Replace third_party with App Engine third_party/. sys.path[i] = os.path.abspath( os.path.join('src', 'appengine', 'third_party')) if sys.version_info.major == 2: # TODO(ochang): Remove once migrated to Python 3. appengine_sdk_path = appengine.find_sdk_path() sys.path.insert(0, appengine_sdk_path) # Get additional App Engine third party imports. import dev_appserver dev_appserver.fix_google_path() sys.path.extend(dev_appserver.EXTRA_PATHS) # Loading appengine_main from the current project ensures that any # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: from src.appengine import main as appengine_main (appengine_main) # pylint: disable=pointless-statement except ImportError: print('Note: unable to import appengine_main.') # google.auth uses App Engine credentials based on importability of # google.appengine.api.app_identity. try: from google.auth import app_engine as auth_app_engine if auth_app_engine.app_identity: auth_app_engine.app_identity = None except ImportError: pass elif args.target == 'core': test_directory = CORE_TEST_DIRECTORY else: # Config module tests. os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir test_directory = os.path.join(args.config_dir, 'modules') top_level_dir = None # Modules may use libs from our App Engine directory. sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) # Fix paths again to get config modules added to the import path. from python.base import modules modules.fix_module_search_paths() # Set expected environment variables. local_config.ProjectConfig().set_environment() # Needed for NDB to work with cloud datastore emulator. os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true' if args.verbose: # Force logging to console for this process and child processes. os.environ['LOG_TO_CONSOLE'] = 'True' else: # Disable logging. logging.disable(logging.CRITICAL) if args.pattern is None: args.pattern = '*_test.py' if args.parallel: # TODO(tanin): Support coverage. run_tests_parallel(args, test_directory, top_level_dir) else: run_tests_single_core(args, test_directory, top_level_dir)
def _update_alerts(project): """Update pubsub topics.""" if local_config.ProjectConfig().get('monitoring.enabled'): _update_deployment_manager(project, 'alerts', os.path.join('monitoring', 'alerts.yaml'))
def execute(args): """Run Python unit tests. For unittests involved appengine, sys.path needs certain modification.""" os.environ["PY_UNITTESTS"] = "True" if os.getenv("INTEGRATION") or os.getenv("UNTRUSTED_RUNNER_TESTS"): # Set up per-user buckets used by integration tests. os.environ["CORPUS_BUCKET"] = common.test_bucket("TEST_CORPUS_BUCKET") os.environ["QUARANTINE_BUCKET"] = common.test_bucket( "TEST_QUARANTINE_BUCKET") os.environ["BACKUP_BUCKET"] = common.test_bucket("TEST_BACKUP_BUCKET") os.environ["COVERAGE_BUCKET"] = common.test_bucket( "TEST_COVERAGE_BUCKET") # Kill leftover instances of emulators and dev appserver. common.kill_leftover_emulators() # Don't use absolute paths to make it easier to compare results in tests. os.environ["CONFIG_DIR_OVERRIDE"] = os.path.join(".", "configs", "test") top_level_dir = os.path.join("src", "python") if args.target == "appengine": # Build template files. appengine.build_templates() test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join("src", "appengine"))) # Get additional App Engine third party imports. import dev_appserver sys.path.extend(dev_appserver.EXTRA_PATHS) # Loading appengine_config from the current project ensures that any # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: from src.appengine import appengine_config (appengine_config) # pylint: disable=pointless-statement except ImportError: print("Note: unable to import appengine_config.") elif args.target == "core": test_directory = CORE_TEST_DIRECTORY else: # Config module tests. os.environ["CONFIG_DIR_OVERRIDE"] = args.config_dir test_directory = os.path.join(args.config_dir, "modules") top_level_dir = None # Modules may use libs from our App Engine directory. sys.path.insert(0, os.path.abspath(os.path.join("src", "appengine"))) # Fix paths again to get config modules added to the import path. from python.base import modules modules.fix_module_search_paths() # Set expected environment variables. local_config.ProjectConfig().set_environment() # Needed for NDB to work with cloud datastore emulator. os.environ["DATASTORE_USE_PROJECT_ID_AS_APP_ID"] = "true" if args.verbose: # Force logging to console for this process and child processes. os.environ["LOG_TO_CONSOLE"] = "True" else: # Disable logging. logging.disable(logging.CRITICAL) if args.pattern is None: args.pattern = "*_test.py" if args.parallel: # TODO(tanin): Support coverage. run_tests_parallel(args, test_directory, top_level_dir) else: run_tests_single_core(args, test_directory, top_level_dir)
def execute(args): """Run Python unit tests. For unittests involved appengine, sys.path needs certain modification.""" os.environ['PY_UNITTESTS'] = 'True' if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'): # Set up per-user buckets used by integration tests. os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET') os.environ['QUARANTINE_BUCKET'] = common.test_bucket( 'TEST_QUARANTINE_BUCKET') os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET') os.environ['COVERAGE_BUCKET'] = common.test_bucket('TEST_COVERAGE_BUCKET') # Kill leftover instances of emulators and dev appserver. common.kill_leftover_emulators() # Don't use absolute paths to make it easier to compare results in tests. os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test') if args.target == 'appengine': # Build template files. appengine.build_templates() test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) # Get additional App Engine third party imports. import dev_appserver sys.path.extend(dev_appserver.EXTRA_PATHS) # Loading appengine_config from the current project ensures that any # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: from src.appengine import appengine_config (appengine_config) # pylint: disable=pointless-statement except ImportError: print('Note: unable to import appengine_config.') else: test_directory = CORE_TEST_DIRECTORY # Set expected environment variables. local_config.ProjectConfig().set_environment() # Needed for NDB to work with cloud datastore emulator. os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true' if args.verbose: logs.configure_for_tests() else: # Disable logging. logging.disable(logging.CRITICAL) enable_coverage = args.pattern is None if args.pattern is None: args.pattern = '*_test.py' if args.parallel: # TODO(tanin): Support coverage. run_tests_parallel(args, test_directory) else: run_tests_single_core(args, test_directory, enable_coverage)
def _update_alerts(project): """Update pubsub topics.""" if local_config.ProjectConfig().get("monitoring.enabled"): _update_deployment_manager(project, "alerts", os.path.join("monitoring", "alerts.yaml"))