def test_pythonmalloc(self):
# Test the PYTHONMALLOC environment variable
pymalloc = support.with_pymalloc()
if pymalloc:
default_name = 'pymalloc_debug' if Py_DEBUG else 'pymalloc'
default_name_debug = 'pymalloc_debug'
else:
default_name = 'malloc_debug' if Py_DEBUG else 'malloc'
default_name_debug = 'malloc_debug'
tests = [
(None, default_name),
('debug', default_name_debug),
('malloc', 'malloc'),
('malloc_debug', 'malloc_debug'),
]
if pymalloc:
tests.extend((
('pymalloc', 'pymalloc'),
('pymalloc_debug', 'pymalloc_debug'),
))
for env_var, name in tests:
with self.subTest(env_var=env_var, name=name):
self.check_pythonmalloc(env_var, name)
def test_getallocatedblocks(self):
try:
import _testcapi
except ImportError:
with_pymalloc = support.with_pymalloc()
else:
alloc_name = _testcapi.pymem_getallocatorsname()
with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug'))
# Some sanity checks
a = sys.getallocatedblocks()
self.assertIs(type(a), int)
if with_pymalloc:
self.assertGreater(a, 0)
else:
# When WITH_PYMALLOC isn't available, we don't know anything
# about the underlying implementation: the function might
# return 0 or something greater.
self.assertGreaterEqual(a, 0)
try:
# While we could imagine a Python session where the number of
# multiple buffer objects would exceed the sharing of references,
# it is unlikely to happen in a normal test run.
self.assertLess(a, sys.gettotalrefcount())
except AttributeError:
# gettotalrefcount() not available
pass
gc.collect()
b = sys.getallocatedblocks()
self.assertLessEqual(b, a)
gc.collect()
c = sys.getallocatedblocks()
self.assertIn(c, range(b - 50, b + 50))
def test_getallocatedblocks(self):
try:
import _testcapi
except ImportError:
with_pymalloc = support.with_pymalloc()
else:
alloc_name = _testcapi.pymem_getallocatorsname()
with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug'))
# Some sanity checks
a = sys.getallocatedblocks()
self.assertIs(type(a), int)
if with_pymalloc:
self.assertGreater(a, 0)
else:
# When WITH_PYMALLOC isn't available, we don't know anything
# about the underlying implementation: the function might
# return 0 or something greater.
self.assertGreaterEqual(a, 0)
try:
# While we could imagine a Python session where the number of
# multiple buffer objects would exceed the sharing of references,
# it is unlikely to happen in a normal test run.
self.assertLess(a, sys.gettotalrefcount())
except AttributeError:
# gettotalrefcount() not available
pass
gc.collect()
b = sys.getallocatedblocks()
self.assertLessEqual(b, a)
gc.collect()
c = sys.getallocatedblocks()
self.assertIn(c, range(b - 50, b + 50))
def test_pythonmalloc(self):
# Test the PYTHONMALLOC environment variable
pymalloc = support.with_pymalloc()
if pymalloc:
default_name = 'pymalloc_debug' if Py_DEBUG else 'pymalloc'
default_name_debug = 'pymalloc_debug'
else:
default_name = 'malloc_debug' if Py_DEBUG else 'malloc'
default_name_debug = 'malloc_debug'
tests = [
(None, default_name),
('debug', default_name_debug),
('malloc', 'malloc'),
('malloc_debug', 'malloc_debug'),
]
if pymalloc:
tests.extend((
('pymalloc', 'pymalloc'),
('pymalloc_debug', 'pymalloc_debug'),
))
for env_var, name in tests:
with self.subTest(env_var=env_var, name=name):
self.check_pythonmalloc(env_var, name)
def test_xdev(self):
# sys.flags.dev_mode
code = "import sys; print(sys.flags.dev_mode)"
out = self.run_xdev("-c", code, xdev=False)
self.assertEqual(out, "False")
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
# Warnings
code = ("import warnings; "
"print(' '.join('%s::%s' % (f[0], f[2].__name__) "
"for f in warnings.filters))")
if Py_DEBUG:
expected_filters = "default::Warning"
else:
expected_filters = ("default::Warning "
"default::DeprecationWarning "
"ignore::DeprecationWarning "
"ignore::PendingDeprecationWarning "
"ignore::ImportWarning "
"ignore::ResourceWarning")
out = self.run_xdev("-c", code)
self.assertEqual(out, expected_filters)
out = self.run_xdev("-b", "-c", code)
self.assertEqual(out, f"default::BytesWarning {expected_filters}")
out = self.run_xdev("-bb", "-c", code)
self.assertEqual(out, f"error::BytesWarning {expected_filters}")
out = self.run_xdev("-Werror", "-c", code)
self.assertEqual(out, f"error::Warning {expected_filters}")
# Memory allocator debug hooks
try:
import _testcapi
except ImportError:
pass
else:
code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())"
with support.SuppressCrashReport():
out = self.run_xdev("-c", code, check_exitcode=False)
if support.with_pymalloc():
alloc_name = "pymalloc_debug"
else:
alloc_name = "malloc_debug"
self.assertEqual(out, alloc_name)
# Faulthandler
try:
import faulthandler
except ImportError:
pass
else:
code = "import faulthandler; print(faulthandler.is_enabled())"
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
def test_xdev(self):
# sys.flags.dev_mode
code = "import sys; print(sys.flags.dev_mode)"
out = self.run_xdev("-c", code, xdev=False)
self.assertEqual(out, "False")
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
# Warnings
code = ("import warnings; "
"print(' '.join('%s::%s' % (f[0], f[2].__name__) "
"for f in warnings.filters))")
if Py_DEBUG:
expected_filters = "default::Warning"
else:
expected_filters = ("default::Warning "
"default::DeprecationWarning "
"ignore::DeprecationWarning "
"ignore::PendingDeprecationWarning "
"ignore::ImportWarning "
"ignore::ResourceWarning")
out = self.run_xdev("-c", code)
self.assertEqual(out, expected_filters)
out = self.run_xdev("-b", "-c", code)
self.assertEqual(out, f"default::BytesWarning {expected_filters}")
out = self.run_xdev("-bb", "-c", code)
self.assertEqual(out, f"error::BytesWarning {expected_filters}")
out = self.run_xdev("-Werror", "-c", code)
self.assertEqual(out, f"error::Warning {expected_filters}")
# Memory allocator debug hooks
try:
import _testcapi
except ImportError:
pass
else:
code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())"
with support.SuppressCrashReport():
out = self.run_xdev("-c", code, check_exitcode=False)
if support.with_pymalloc():
alloc_name = "pymalloc_debug"
else:
alloc_name = "malloc_debug"
self.assertEqual(out, alloc_name)
# Faulthandler
try:
import faulthandler
except ImportError:
pass
else:
code = "import faulthandler; print(faulthandler.is_enabled())"
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
def test_xdev(self):
code = ("import sys, warnings; "
"print(' '.join('%s::%s' % (f[0], f[2].__name__) "
"for f in warnings.filters))")
out = self.run_xdev("-c", code)
self.assertEqual(
out, "ignore::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-b", "-c", code)
self.assertEqual(
out, "default::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-bb", "-c", code)
self.assertEqual(
out, "error::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-Werror", "-c", code)
self.assertEqual(
out, "error::Warning "
"ignore::BytesWarning "
"default::ResourceWarning "
"default::Warning")
try:
import _testcapi
except ImportError:
pass
else:
code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())"
with support.SuppressCrashReport():
out = self.run_xdev("-c", code, check_exitcode=False)
if support.with_pymalloc():
alloc_name = "pymalloc_debug"
else:
alloc_name = "malloc_debug"
self.assertEqual(out, alloc_name)
try:
import faulthandler
except ImportError:
pass
else:
code = "import faulthandler; print(faulthandler.is_enabled())"
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
def test_xdev(self):
code = ("import sys, warnings; "
"print(' '.join('%s::%s' % (f[0], f[2].__name__) "
"for f in warnings.filters))")
out = self.run_xdev("-c", code)
self.assertEqual(out,
"ignore::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-b", "-c", code)
self.assertEqual(out,
"default::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-bb", "-c", code)
self.assertEqual(out,
"error::BytesWarning "
"default::ResourceWarning "
"default::Warning")
out = self.run_xdev("-Werror", "-c", code)
self.assertEqual(out,
"error::Warning "
"ignore::BytesWarning "
"default::ResourceWarning "
"default::Warning")
try:
import _testcapi
except ImportError:
pass
else:
code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())"
with support.SuppressCrashReport():
out = self.run_xdev("-c", code, check_exitcode=False)
if support.with_pymalloc():
alloc_name = "pymalloc_debug"
else:
alloc_name = "malloc_debug"
self.assertEqual(out, alloc_name)
try:
import faulthandler
except ImportError:
pass
else:
code = "import faulthandler; print(faulthandler.is_enabled())"
out = self.run_xdev("-c", code)
self.assertEqual(out, "True")
def test_pyobject_null_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_null_is_freed')
def test_pyobject_uninitialized_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
def test_pyobject_forbidden_bytes_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
def test_pyobject_freed_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
class PyMemMallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'malloc_debug'
@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
@unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG')
class PyMemDefaultTests(PyMemDebugTests):
# test default allocator of Python compiled in debug mode
PYTHONMALLOC = ''
if __name__ == "__main__":
unittest.main()
def test_pymem_malloc_without_gil(self):
# Debug hooks must raise an error if PyMem_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
self.check_malloc_without_gil(code)
def test_pyobject_malloc_without_gil(self):
# Debug hooks must raise an error if PyObject_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
self.check_malloc_without_gil(code)
class PyMemMallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'malloc_debug'
@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
@unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG')
class PyMemDefaultTests(PyMemDebugTests):
# test default allocator of Python compiled in debug mode
PYTHONMALLOC = ''
if __name__ == "__main__":
unittest.main()
class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
maxDiff = 4096
UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape')
# core config
UNTESTED_CORE_CONFIG = (
# FIXME: untested core configuration variables
'dll_path',
'executable',
'module_search_paths',
)
# Mark config which should be get by get_default_config()
GET_DEFAULT_CONFIG = object()
DEFAULT_CORE_CONFIG = {
'install_signal_handlers': 1,
'use_environment': 1,
'use_hash_seed': 0,
'hash_seed': 0,
'allocator': None,
'dev_mode': 0,
'faulthandler': 0,
'tracemalloc': 0,
'import_time': 0,
'show_ref_count': 0,
'show_alloc_count': 0,
'dump_refs': 0,
'malloc_stats': 0,
'filesystem_encoding': GET_DEFAULT_CONFIG,
'filesystem_errors': GET_DEFAULT_CONFIG,
'utf8_mode': 0,
'coerce_c_locale': 0,
'coerce_c_locale_warn': 0,
'pycache_prefix': None,
'program_name': './_testembed',
'argv': [],
'program': None,
'xoptions': [],
'warnoptions': [],
'module_search_path_env': None,
'home': None,
'prefix': GET_DEFAULT_CONFIG,
'base_prefix': GET_DEFAULT_CONFIG,
'exec_prefix': GET_DEFAULT_CONFIG,
'base_exec_prefix': GET_DEFAULT_CONFIG,
'isolated': 0,
'site_import': 1,
'bytes_warning': 0,
'inspect': 0,
'interactive': 0,
'optimization_level': 0,
'parser_debug': 0,
'write_bytecode': 1,
'verbose': 0,
'quiet': 0,
'user_site_directory': 1,
'buffered_stdio': 1,
'stdio_encoding': GET_DEFAULT_CONFIG,
'stdio_errors': GET_DEFAULT_CONFIG,
'skip_source_first_line': 0,
'run_command': None,
'run_module': None,
'run_filename': None,
'_install_importlib': 1,
'_check_hash_pycs_mode': 'default',
'_frozen': 0,
}
if MS_WINDOWS:
DEFAULT_CORE_CONFIG.update({
'legacy_windows_fs_encoding': 0,
'legacy_windows_stdio': 0,
})
DEBUG_ALLOCATOR = 'pymalloc_debug' if support.with_pymalloc(
) else 'malloc_debug'
# main config
COPY_MAIN_CONFIG = (
# Copy core config to main config for expected values
'argv',
'base_exec_prefix',
'base_prefix',
'exec_prefix',
'executable',
'install_signal_handlers',
'prefix',
'pycache_prefix',
'warnoptions',
# xoptions is created from core_config in check_main_config().
# 'module_search_paths' is copied to 'module_search_path'.
)
# global config
DEFAULT_GLOBAL_CONFIG = {
'Py_HasFileSystemDefaultEncoding': 0,
'Py_HashRandomizationFlag': 1,
'_Py_HasFileSystemDefaultEncodeErrors': 0,
}
COPY_GLOBAL_CONFIG = [
# Copy core config to global config for expected values
# True means that the core config value is inverted (0 => 1 and 1 => 0)
('Py_BytesWarningFlag', 'bytes_warning'),
('Py_DebugFlag', 'parser_debug'),
('Py_DontWriteBytecodeFlag', 'write_bytecode', True),
('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
('Py_FrozenFlag', '_frozen'),
('Py_IgnoreEnvironmentFlag', 'use_environment', True),
('Py_InspectFlag', 'inspect'),
('Py_InteractiveFlag', 'interactive'),
('Py_IsolatedFlag', 'isolated'),
('Py_NoSiteFlag', 'site_import', True),
('Py_NoUserSiteDirectory', 'user_site_directory', True),
('Py_OptimizeFlag', 'optimization_level'),
('Py_QuietFlag', 'quiet'),
('Py_UTF8Mode', 'utf8_mode'),
('Py_UnbufferedStdioFlag', 'buffered_stdio', True),
('Py_VerboseFlag', 'verbose'),
]
if MS_WINDOWS:
COPY_GLOBAL_CONFIG.extend((
('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'),
('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
))
def main_xoptions(self, xoptions_list):
xoptions = {}
for opt in xoptions_list:
if '=' in opt:
key, value = opt.split('=', 1)
xoptions[key] = value
else:
xoptions[opt] = True
return xoptions
def check_main_config(self, config):
core_config = config['core_config']
main_config = config['main_config']
# main config
expected = {}
for key in self.COPY_MAIN_CONFIG:
expected[key] = core_config[key]
expected['module_search_path'] = core_config['module_search_paths']
expected['xoptions'] = self.main_xoptions(core_config['xoptions'])
self.assertEqual(main_config, expected)
def get_expected_config(self, expected, env):
expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
code = textwrap.dedent('''
import json
import sys
data = {
'stdio_encoding': sys.stdout.encoding,
'stdio_errors': sys.stdout.errors,
'prefix': sys.prefix,
'base_prefix': sys.base_prefix,
'exec_prefix': sys.exec_prefix,
'base_exec_prefix': sys.base_exec_prefix,
'filesystem_encoding': sys.getfilesystemencoding(),
'filesystem_errors': sys.getfilesystemencodeerrors(),
}
data = json.dumps(data)
data = data.encode('utf-8')
sys.stdout.buffer.write(data)
sys.stdout.buffer.flush()
''')
# Use -S to not import the site module: get the proper configuration
# when test_embed is run from a venv (bpo-35313)
args = (sys.executable, '-S', '-c', code)
env = dict(env)
if not expected['isolated']:
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
proc = subprocess.run(args,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if proc.returncode:
raise Exception(f"failed to get the default config: "
f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
stdout = proc.stdout.decode('utf-8')
config = json.loads(stdout)
for key, value in expected.items():
if value is self.GET_DEFAULT_CONFIG:
expected[key] = config[key]
return expected
def check_core_config(self, config, expected):
core_config = dict(config['core_config'])
for key in self.UNTESTED_CORE_CONFIG:
core_config.pop(key, None)
self.assertEqual(core_config, expected)
def check_global_config(self, config):
core_config = config['core_config']
expected = dict(self.DEFAULT_GLOBAL_CONFIG)
for item in self.COPY_GLOBAL_CONFIG:
if len(item) == 3:
global_key, core_key, opposite = item
expected[global_key] = 0 if core_config[core_key] else 1
else:
global_key, core_key = item
expected[global_key] = core_config[core_key]
self.assertEqual(config['global_config'], expected)
def check_config(self, testname, expected):
env = dict(os.environ)
# Remove PYTHON* environment variables to get deterministic environment
for key in list(env):
if key.startswith('PYTHON'):
del env[key]
# Disable C locale coercion and UTF-8 mode to not depend
# on the current locale
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
out, err = self.run_embedded_interpreter(testname, env=env)
# Ignore err
config = json.loads(out)
expected = self.get_expected_config(expected, env)
self.check_core_config(config, expected)
self.check_main_config(config)
self.check_global_config(config)
def test_init_default_config(self):
self.check_config("init_default_config", {})
def test_init_global_config(self):
config = {
'program_name': './globalvar',
'site_import': 0,
'bytes_warning': 1,
'inspect': 1,
'interactive': 1,
'optimization_level': 2,
'write_bytecode': 0,
'verbose': 1,
'quiet': 1,
'buffered_stdio': 0,
'utf8_mode': 1,
'stdio_encoding': 'utf-8',
'stdio_errors': 'surrogateescape',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'user_site_directory': 0,
'_frozen': 1,
}
self.check_config("init_global_config", config)
def test_init_from_config(self):
config = {
'install_signal_handlers': 0,
'use_hash_seed': 1,
'hash_seed': 123,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'show_ref_count': 1,
'show_alloc_count': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'pycache_prefix': 'conf_pycache_prefix',
'program_name': './conf_program_name',
'argv': ['-c', 'pass'],
'program': 'conf_program',
'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
'warnoptions': ['default', 'error::ResourceWarning'],
'site_import': 0,
'bytes_warning': 1,
'inspect': 1,
'interactive': 1,
'optimization_level': 2,
'write_bytecode': 0,
'verbose': 1,
'quiet': 1,
'buffered_stdio': 0,
'user_site_directory': 0,
'faulthandler': 1,
'_check_hash_pycs_mode': 'always',
'_frozen': 1,
}
self.check_config("init_from_config", config)
INIT_ENV_CONFIG = {
'use_hash_seed': 1,
'hash_seed': 42,
'allocator': 'malloc_debug',
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'utf8_mode': 1,
'filesystem_encoding': 'utf-8',
'filesystem_errors': UTF8_MODE_ERRORS,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
}
def test_init_env(self):
self.check_config("init_env", self.INIT_ENV_CONFIG)
def test_init_env_dev_mode(self):
config = dict(self.INIT_ENV_CONFIG,
allocator=self.DEBUG_ALLOCATOR,
dev_mode=1)
self.check_config("init_env_dev_mode", config)
def test_init_dev_mode(self):
config = {
'dev_mode': 1,
'faulthandler': 1,
'allocator': self.DEBUG_ALLOCATOR,
}
self.check_config("init_dev_mode", config)
def test_init_isolated(self):
config = {
'isolated': 1,
'use_environment': 0,
'user_site_directory': 0,
}
self.check_config("init_isolated", config)
