def fix_module_search_paths(): """Add directories that we must be able to import from to path.""" root_directory = os.environ['ROOT_DIR'] source_directory = os.path.join(root_directory, 'src') if sys.platform.startswith('win'): pythonpath_sep = ';' else: pythonpath_sep = ':' python_path = os.getenv('PYTHONPATH', '').split(pythonpath_sep) third_party_libraries_directory = os.path.join(source_directory, 'third_party') if third_party_libraries_directory not in sys.path: sys.path.insert(0, third_party_libraries_directory) python_path.insert(0, third_party_libraries_directory) python_source_directory = os.path.join(source_directory, 'python') if python_source_directory not in sys.path: sys.path.insert(0, python_source_directory) python_path.insert(0, python_source_directory) if source_directory not in sys.path: sys.path.insert(0, source_directory) python_path.insert(0, source_directory) # Since paths have now been fixed, import the custom logs module. from metrics import logs # Work around google package issues with App Engine SDK. try: import dev_appserver dev_appserver.fix_google_path() except (ImportError, RuntimeError) as error: logs.log_error('Fixing google import failed: %s' % error) # Make protobuf compatible with appengine. # FIXME(unassigned): remove try-except once we confirm it works. try: import google protobuf_directory = os.path.join(third_party_libraries_directory, 'google') google.__path__.append(protobuf_directory) except (ImportError, RuntimeError) as error: logs.log_error('Fixing protobuf-appengine compatibility failed: %s' % error) os.environ['PYTHONPATH'] = pythonpath_sep.join(python_path) # Add site directory to make from imports work in google namespace. site.addsitedir(third_party_libraries_directory) disable_known_module_warnings() _patch_appengine_modules_for_bots()
def fix_paths(auto_download=False, default_dir=DEFAULT_DIR, version=DEFAULT_VERSION, index=-1, include_internal_imports=False): if _is_on_appengine() or _is_on_dev_appserver(): return sdk_path = _find_or_download_sdk(auto_download, default_dir, version) assert sdk_path is not None, _get_error_message(version, default_dir) os.environ['APPENGINE_SDK'] = sdk_path if index < 0: # e.g., index == -1 means to add paths to the very end. index = len(sys.path) + index + 1 if sdk_path not in sys.path: sys.path.insert(index, sdk_path) index += 1 old_cwd = os.getcwd() import dev_appserver dev_appserver.fix_google_path() new_cwd = os.getcwd() if new_cwd != old_cwd: # Workaround for google issue 117145272. # (See https://issuetracker.google.com/issues/117145272 ) os.chdir(old_cwd) more_paths = dev_appserver.EXTRA_PATHS if include_internal_imports: more_paths += dev_appserver._PATHS.script_paths( 'dev_appserver.py') # for cherrypy, portpicker, etc. more_paths = [x for x in more_paths if x not in sys.path] sys.path[index:index] = more_paths
def execute(args): """Run Python unit tests. For unittests involved appengine, sys.path needs certain modification.""" os.environ['PY_UNITTESTS'] = 'True' os.environ['CLOUDSDK_PYTHON'] = 'python2' if os.getenv('INTEGRATION') or os.getenv('UNTRUSTED_RUNNER_TESTS'): # Set up per-user buckets used by integration tests. os.environ['CORPUS_BUCKET'] = common.test_bucket('TEST_CORPUS_BUCKET') os.environ['QUARANTINE_BUCKET'] = common.test_bucket( 'TEST_QUARANTINE_BUCKET') os.environ['BACKUP_BUCKET'] = common.test_bucket('TEST_BACKUP_BUCKET') os.environ['COVERAGE_BUCKET'] = common.test_bucket( 'TEST_COVERAGE_BUCKET') # Kill leftover instances of emulators and dev appserver. common.kill_leftover_emulators() # Don't use absolute paths to make it easier to compare results in tests. os.environ['CONFIG_DIR_OVERRIDE'] = os.path.join('.', 'configs', 'test') top_level_dir = os.path.join('src', 'python') if args.target == 'appengine': # Build template files. appengine.build_templates() test_directory = APPENGINE_TEST_DIRECTORY sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) for i, path in enumerate(sys.path): if 'third_party' in path: # Replace third_party with App Engine third_party/. sys.path[i] = os.path.abspath( os.path.join('src', 'appengine', 'third_party')) if sys.version_info.major == 2: # TODO(ochang): Remove once migrated to Python 3. appengine_sdk_path = appengine.find_sdk_path() sys.path.insert(0, appengine_sdk_path) # Get additional App Engine third party imports. import dev_appserver dev_appserver.fix_google_path() sys.path.extend(dev_appserver.EXTRA_PATHS) # Loading appengine_main from the current project ensures that any # changes to configuration there are available to all tests (e.g. # sys.path modifications, namespaces, etc.) try: from src.appengine import main as appengine_main (appengine_main) # pylint: disable=pointless-statement except ImportError: print('Note: unable to import appengine_main.') # google.auth uses App Engine credentials based on importability of # google.appengine.api.app_identity. try: from google.auth import app_engine as auth_app_engine if auth_app_engine.app_identity: auth_app_engine.app_identity = None except ImportError: pass elif args.target == 'core': test_directory = CORE_TEST_DIRECTORY else: # Config module tests. os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir test_directory = os.path.join(args.config_dir, 'modules') top_level_dir = None # Modules may use libs from our App Engine directory. sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine'))) # Fix paths again to get config modules added to the import path. from python.base import modules modules.fix_module_search_paths() # Set expected environment variables. local_config.ProjectConfig().set_environment() # Needed for NDB to work with cloud datastore emulator. os.environ['DATASTORE_USE_PROJECT_ID_AS_APP_ID'] = 'true' if args.verbose: # Force logging to console for this process and child processes. os.environ['LOG_TO_CONSOLE'] = 'True' else: # Disable logging. logging.disable(logging.CRITICAL) if args.pattern is None: args.pattern = '*_test.py' if args.parallel: # TODO(tanin): Support coverage. run_tests_parallel(args, test_directory, top_level_dir) else: run_tests_single_core(args, test_directory, top_level_dir)