def fix_module_search_paths():
"""Add directories that we must be able to import from to path."""
root_directory = os.environ['ROOT_DIR']
source_directory = os.path.join(root_directory, 'src')
if sys.platform.startswith('win'):
pythonpath_sep = ';'
else:
pythonpath_sep = ':'
python_path = os.getenv('PYTHONPATH', '').split(pythonpath_sep)
third_party_libraries_directory = os.path.join(source_directory,
'third_party')
if third_party_libraries_directory not in sys.path:
sys.path.insert(0, third_party_libraries_directory)
python_path.insert(0, third_party_libraries_directory)
python_source_directory = os.path.join(source_directory, 'python')
if python_source_directory not in sys.path:
sys.path.insert(0, python_source_directory)
python_path.insert(0, python_source_directory)
if source_directory not in sys.path:
sys.path.insert(0, source_directory)
python_path.insert(0, source_directory)
# Since paths have now been fixed, import the custom logs module.
from metrics import logs
# Work around google package issues with App Engine SDK.
try:
import dev_appserver
dev_appserver.fix_google_path()
except (ImportError, RuntimeError) as error:
logs.log_error('Fixing google import failed: %s' % error)
# Make protobuf compatible with appengine.
# FIXME(unassigned): remove try-except once we confirm it works.
try:
import google
protobuf_directory = os.path.join(third_party_libraries_directory,
'google')
google.__path__.append(protobuf_directory)
except (ImportError, RuntimeError) as error:
logs.log_error('Fixing protobuf-appengine compatibility failed: %s' %
error)
os.environ['PYTHONPATH'] = pythonpath_sep.join(python_path)
# Add site directory to make from imports work in google namespace.
site.addsitedir(third_party_libraries_directory)
disable_known_module_warnings()
_patch_appengine_modules_for_bots()
def fix_paths(auto_download=False,
default_dir=DEFAULT_DIR,
version=DEFAULT_VERSION,
index=-1,
include_internal_imports=False):
if _is_on_appengine() or _is_on_dev_appserver():
return
sdk_path = _find_or_download_sdk(auto_download, default_dir, version)
assert sdk_path is not None, _get_error_message(version, default_dir)
os.environ['APPENGINE_SDK'] = sdk_path
if index < 0:
# e.g., index == -1 means to add paths to the very end.
index = len(sys.path) + index + 1
if sdk_path not in sys.path:
sys.path.insert(index, sdk_path)
index += 1
old_cwd = os.getcwd()
import dev_appserver
dev_appserver.fix_google_path()
new_cwd = os.getcwd()
if new_cwd != old_cwd:
# Workaround for google issue 117145272.
# (See https://issuetracker.google.com/issues/117145272 )
os.chdir(old_cwd)
more_paths = dev_appserver.EXTRA_PATHS
if include_internal_imports:
more_paths += dev_appserver._PATHS.script_paths(
'dev_appserver.py') # for cherrypy, portpicker, etc.
more_paths = [x for x in more_paths if x not in sys.path]
sys.path[index:index] = more_paths
def execute(args):
"""Run Python unit tests. For unittests involved appengine, sys.path needs
certain modification."""
os.environ['PY_UNITTESTS'] = 'True'
os.environ['CLOUDSDK_PYTHON'] = 'python2'
if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'):
# Set up per-user buckets used by integration tests.
os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET')
os.environ['QUARANTINE_BUCKET'] = common.test_bucket(
'TEST_QUARANTINE_BUCKET')
os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET')
os.environ['COVERAGE_BUCKET'] = common.test_bucket(
'TEST_COVERAGE_BUCKET')
# Kill leftover instances of emulators and dev appserver.
common.kill_leftover_emulators()
# Don't use absolute paths to make it easier to compare results in tests.
os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test')
top_level_dir = os.path.join('src', 'python')
if args.target == 'appengine':
# Build template files.
appengine.build_templates()
test_directory = APPENGINE_TEST_DIRECTORY
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
for i, path in enumerate(sys.path):
if 'third_party' in path:
# Replace third_party with App Engine third_party/.
sys.path[i] = os.path.abspath(
os.path.join('src', 'appengine', 'third_party'))
if sys.version_info.major == 2:
# TODO(ochang): Remove once migrated to Python 3.
appengine_sdk_path = appengine.find_sdk_path()
sys.path.insert(0, appengine_sdk_path)
# Get additional App Engine third party imports.
import dev_appserver
dev_appserver.fix_google_path()
sys.path.extend(dev_appserver.EXTRA_PATHS)
# Loading appengine_main from the current project ensures that any
# changes to configuration there are available to all tests (e.g.
# sys.path modifications, namespaces, etc.)
try:
from src.appengine import main as appengine_main
(appengine_main) # pylint: disable=pointless-statement
except ImportError:
print('Note: unable to import appengine_main.')
# google.auth uses App Engine credentials based on importability of
# google.appengine.api.app_identity.
try:
from google.auth import app_engine as auth_app_engine
if auth_app_engine.app_identity:
auth_app_engine.app_identity = None
except ImportError:
pass
elif args.target == 'core':
test_directory = CORE_TEST_DIRECTORY
else:
# Config module tests.
os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
test_directory = os.path.join(args.config_dir, 'modules')
top_level_dir = None
# Modules may use libs from our App Engine directory.
sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
# Fix paths again to get config modules added to the import path.
from python.base import modules
modules.fix_module_search_paths()
# Set expected environment variables.
local_config.ProjectConfig().set_environment()
# Needed for NDB to work with cloud datastore emulator.
os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true'
if args.verbose:
# Force logging to console for this process and child processes.
os.environ['LOG_TO_CONSOLE'] = 'True'
else:
# Disable logging.
logging.disable(logging.CRITICAL)
if args.pattern is None:
args.pattern = '*_test.py'
if args.parallel:
# TODO(tanin): Support coverage.
run_tests_parallel(args, test_directory, top_level_dir)
else:
run_tests_single_core(args, test_directory, top_level_dir)
