def repl():
user_ns = {}
py_main = __res.find('PY_MAIN')
if py_main:
mod_name, func_name = py_main.split(':', 1)
try:
import importlib
mod = importlib.import_module(mod_name)
user_ns = mod.__dict__
except:
import traceback
traceback.print_exc()
if py_main and '__main__' not in user_ns:
def run(args):
if isinstance(args, basestring):
import shlex
args = shlex.split(args)
import sys
sys.argv = [sys.argv[0]] + args
getattr(mod, func_name)()
user_ns['__main__'] = run
try:
import IPython
except ImportError:
import code
code.interact(local=user_ns)
else:
IPython.start_ipython(user_ns=user_ns)
def repl():
user_ns = {}
py_main = __res.find('PY_MAIN')
if py_main:
mod_name, func_name = (py_main.split(b':', 1) + [None])[:2]
try:
import importlib
mod = importlib.import_module(mod_name.decode('UTF-8'))
user_ns = mod.__dict__
except:
import traceback
traceback.print_exc()
if func_name and '__main__' not in user_ns:
def run(args):
if isinstance(args, basestring):
import shlex
args = shlex.split(args)
import sys
sys.argv = [sys.argv[0]] + args
getattr(mod, func_name)()
user_ns['__main__'] = run
try:
import IPython
except ModuleNotFoundError:
pass
else:
return IPython.start_ipython(user_ns=user_ns)
import code
code.interact(local=user_ns)
def repl():
user_ns = {}
py_main = __res.find('PY_MAIN')
if py_main:
mod_name, func_name = py_main.split(':', 1)
try:
import importlib
mod = importlib.import_module(mod_name)
user_ns = mod.__dict__
except:
import traceback
traceback.print_exc()
if py_main and '__main__' not in user_ns:
def run(args):
if isinstance(args, basestring):
import shlex
args = shlex.split(args)
import sys
sys.argv = [sys.argv[0]] + args
getattr(mod, func_name)()
user_ns['__main__'] = run
try:
import IPython
except ImportError:
import code
code.interact(local=user_ns)
else:
IPython.start_ipython(user_ns=user_ns)
def main():
skip_names = sys.argv[1:]
os.environ['Y_PYTHON_IMPORT_TEST'] = ''
# We should initialize Django before importing any applications
if os.getenv('DJANGO_SETTINGS_MODULE'):
try:
import django
except ImportError:
pass
else:
django.setup()
py_main = __res.find('PY_MAIN')
if py_main:
py_main_module = py_main.split(b':', 1)[0].decode('UTF-8')
else:
py_main_module = None
try:
check_imports(no_check=skip_names, py_main=py_main_module)
except:
sys.exit(1)
def builtin_cadata():
global _builtin_cadata
if _builtin_cadata is None:
import __res
data = __res.find(b'/builtin/cacert')
# load_verify_locations expects PEM cadata to be an ASCII-only unicode
# object, so we discard unicode in comments.
_builtin_cadata = data.decode('ASCII', errors='ignore')
return _builtin_cadata
def check_imports(no_check=(), extra=(), skip_func=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
str_ = lambda s: s
if not isinstance(b'', str):
str_ = lambda s: s.decode('UTF-8')
exceptions = list(no_check)
for key, _ in __res.iter_keys(b'py/no_check_imports/'):
exceptions += str_(__res.find(key)).split()
if exceptions:
exceptions.sort()
print('NO_CHECK_IMPORTS', ' '.join(exceptions))
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
norm = lambda s: s[:-9] if s.endswith('.__init__') else s
for module in sorted(sys.extra_modules | set(extra), key=norm):
if module not in extra and (rx.search(module)
or skip_func and skip_func(module)):
print('SKIP', module)
continue
name = module.rsplit('.', 1)[-1]
if name == '__main__' and 'if __name__ ==' not in importer.get_source(
module):
print('SKIP', module,
'''without "if __name__ == '__main__'" check''')
continue
try:
print('TRY', module)
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
print('OK ', module)
except Exception as e:
print('FAIL', module, e, file=sys.stderr)
traceback.print_exception(*sys.exc_info())
failed.append('{}: {}'.format(module, e))
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
def open_resource(name):
"""Open a resource from the zoneinfo for reading.
Uses compiled-in timezone files
"""
import __res
from cStringIO import StringIO
res_name = '/pytz_data/zoneinfo/' + name.lstrip('/')
res_buf = __res.find(res_name)
if res_buf is None:
raise IOError("No such resource: " + res_name)
return StringIO(res_buf)
def open_resource(name):
"""Open a resource from the zoneinfo for reading.
Uses compiled-in timezone files
"""
import __res
from cStringIO import StringIO
res_name = '/pytz_data/zoneinfo/' + name.lstrip('/')
res_buf = __res.find(res_name)
if res_buf is None:
raise IOError("No such resource: " + res_name)
return StringIO(res_buf)
def repl():
user_ns = {}
py_main = __res.find('PY_MAIN')
if py_main:
py_main_split = py_main.split(':', 1)
if len(py_main_split) == 2:
mod_name, func_name = py_main_split
else:
mod_name, func_name = py_main_split[0], 'main'
if not mod_name:
mod_name = 'library.python.runtime.entry_points'
try:
import importlib
mod = importlib.import_module(mod_name)
user_ns = mod.__dict__
except:
import traceback
traceback.print_exc()
if '__main__' not in user_ns:
def run(args):
if isinstance(args, basestring):
import shlex
args = shlex.split(args)
import sys
sys.argv = [sys.argv[0]] + args
getattr(mod, func_name)()
user_ns['__main__'] = run
else:
try:
mod = __res.importer.load_module('__main__',
fix_name='__main_real')
user_ns = mod.__dict__
except ImportError:
pass
try:
import IPython
except ImportError:
import code
code.interact(local=user_ns)
else:
IPython.start_ipython(user_ns=user_ns)
def _file_contents(file_name):
if getattr(sys, 'is_standalone_binary', False):
import __res
for prefix in ['/coredump_filter_data/', '']:
res_name = prefix + file_name
data = __res.find(res_name)
if data is not None:
return data
raise IOError("Failed to find resource: " + file_name)
else:
if not os.path.exists(file_name):
file_name = os.path.join(MY_PATH, file_name)
with open(file_name) as f:
return f.read()
def iteritems(prefix='', strip_prefix=False):
for key in iterkeys(prefix=prefix):
value = find(key)
if strip_prefix:
key = key[len(prefix):]
yield key, value
def run_constructors():
for key, module_name in __res.iter_keys(b'py/constructors/'):
import importlib
module = importlib.import_module(module_name.decode())
init_func = getattr(module, __res.find(key).decode())
init_func()
def updatecache(filename, module_globals=None):
"""Update a cache entry and return its list of lines.
If something's wrong, print a message, discard the cache entry,
and return an empty list."""
if filename in cache:
del cache[filename]
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
if not os.path.isabs(filename):
# mod.__loader__.get_source works, but when mod fails to import
# mod.__loader__ and other mod globals are already None.
import __res
key = __res.importer.file_source(filename)
if key:
data = __res.find(key)
assert data is not None, filename
lines = [line + '\n' for line in data.splitlines()]
cache[filename] = (len(data), None, lines, filename)
return cache[filename][2]
fullname = filename
try:
stat = os.stat(fullname)
except OSError:
basename = filename
# Try for a __loader__, if available
if module_globals and '__loader__' in module_globals:
name = module_globals.get('__name__')
loader = module_globals['__loader__']
get_source = getattr(loader, 'get_source', None)
if name and get_source:
try:
data = get_source(name)
except (ImportError, IOError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (
len(data), None,
[line+'\n' for line in data.splitlines()], fullname
)
return cache[filename][2]
# Try looking through the module search path, which is only useful
# when handling a relative filename.
if os.path.isabs(filename):
return []
for dirname in sys.path:
# When using imputil, sys.path may contain things other than
# strings; ignore them when it happens.
try:
fullname = os.path.join(dirname, basename)
except (TypeError, AttributeError):
# Not sufficiently string-like to do anything useful with.
continue
try:
stat = os.stat(fullname)
break
except os.error:
pass
else:
return []
try:
with open(fullname, 'rU') as fp:
lines = fp.readlines()
except IOError:
return []
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
size, mtime = stat.st_size, stat.st_mtime
cache[filename] = size, mtime, lines, fullname
return lines
def updatecache(filename, module_globals=None):
"""Update a cache entry and return its list of lines.
If something's wrong, print a message, discard the cache entry,
and return an empty list."""
if filename in cache:
if len(cache[filename]) != 1:
cache.pop(filename, None)
if not filename or (filename.startswith('<') and filename.endswith('>')):
return []
if not os.path.isabs(filename):
# Do not read builtin code from the filesystem.
import __res
key = __res.importer.file_source(filename)
if key:
data = __res.find(key)
assert data is not None, filename
data = data.decode('UTF-8')
lines = [line + '\n' for line in data.splitlines()]
cache[filename] = (len(data), None, lines, filename)
return cache[filename][2]
fullname = filename
try:
stat = os.stat(fullname)
except OSError:
basename = filename
# Realise a lazy loader based lookup if there is one
# otherwise try to lookup right now.
if lazycache(filename, module_globals):
try:
data = cache[filename][0]()
except (ImportError, OSError):
pass
else:
if data is None:
# No luck, the PEP302 loader cannot find the source
# for this module.
return []
cache[filename] = (len(data), None,
[line + '\n'
for line in data.splitlines()], fullname)
return cache[filename][2]
# Try looking through the module search path, which is only useful
# when handling a relative filename.
if os.path.isabs(filename):
return []
for dirname in sys.path:
try:
fullname = os.path.join(dirname, basename)
except (TypeError, AttributeError):
# Not sufficiently string-like to do anything useful with.
continue
try:
stat = os.stat(fullname)
break
except OSError:
pass
else:
return []
try:
with tokenize.open(fullname) as fp:
lines = fp.readlines()
except OSError:
return []
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
size, mtime = stat.st_size, stat.st_mtime
cache[filename] = size, mtime, lines, fullname
return lines
def itervalues(prefix=b''):
for key in iterkeys(prefix=prefix):
value = find(key)
yield value
def main():
import library.python.pytest.context as context
context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time()
profile = None
if '--profile-pytest' in sys.argv:
sys.argv.remove('--profile-pytest')
import pstats
import cProfile
profile = cProfile.Profile()
profile.enable()
import pytest
import library.python.pytest.plugins.collection as collection
import library.python.pytest.plugins.ya as ya
import library.python.pytest.plugins.conftests as conftests
import _pytest.assertion
from _pytest.monkeypatch import MonkeyPatch
from . import rewrite
m = MonkeyPatch()
m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook",
rewrite.AssertionRewritingHook)
prefix = '__tests__.'
test_modules = [
name[len(prefix):] for name in sys.extra_modules
if name.startswith(prefix) and not name.endswith('.conftest')
]
doctest_packages = (__res.find("PY_DOCTEST_PACKAGES") or "").split()
def is_doctest_module(name):
for package in doctest_packages:
if name == package or name.startswith(package + "."):
return True
return False
doctest_modules = [
name for name in sys.extra_modules if is_doctest_module(name)
]
def remove_user_site(paths):
site_paths = ('site-packages', 'site-python')
def is_site_path(path):
for p in site_paths:
if path.find(p) != -1:
return True
return False
new_paths = list(paths)
for p in paths:
if is_site_path(p):
new_paths.remove(p)
return new_paths
sys.path = remove_user_site(sys.path)
rc = pytest.main(plugins=[
collection.CollectionPlugin(test_modules, doctest_modules),
ya,
conftests,
])
if rc == 5:
# don't care about EXIT_NOTESTSCOLLECTED
rc = 0
if profile:
profile.disable()
ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative')
ps.print_stats()
sys.exit(rc)
def main():
import library.python.pytest.context as context
context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time()
profile = None
if '--profile-pytest' in sys.argv:
sys.argv.remove('--profile-pytest')
import pstats
import cProfile
profile = cProfile.Profile()
profile.enable()
# Reset influencing env. vars
# For more info see library/python/testing/yatest_common/yatest/common/errors.py
if FORCE_EXIT_TESTSFAILED_ENV in os.environ:
del os.environ[FORCE_EXIT_TESTSFAILED_ENV]
if "Y_PYTHON_CLEAR_ENTRY_POINT" in os.environ:
if "Y_PYTHON_ENTRY_POINT" in os.environ:
del os.environ["Y_PYTHON_ENTRY_POINT"]
del os.environ["Y_PYTHON_CLEAR_ENTRY_POINT"]
listing_mode = '--collect-only' in sys.argv
yatest_runner = os.environ.get('YA_TEST_RUNNER') == '1'
import pytest
import library.python.pytest.plugins.collection as collection
import library.python.pytest.plugins.ya as ya
import library.python.pytest.plugins.conftests as conftests
import _pytest.assertion
from _pytest.monkeypatch import MonkeyPatch
from . import rewrite
m = MonkeyPatch()
m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook",
rewrite.AssertionRewritingHook)
prefix = '__tests__.'
test_modules = [
name[len(prefix):] for name in sys.extra_modules
if name.startswith(prefix) and not name.endswith('.conftest')
]
doctest_packages = __res.find("PY_DOCTEST_PACKAGES") or ""
if isinstance(doctest_packages, bytes):
doctest_packages = doctest_packages.decode('utf-8')
doctest_packages = doctest_packages.split()
def is_doctest_module(name):
for package in doctest_packages:
if name == package or name.startswith(str(package) + "."):
return True
return False
doctest_modules = [
name for name in sys.extra_modules if is_doctest_module(name)
]
def remove_user_site(paths):
site_paths = ('site-packages', 'site-python')
def is_site_path(path):
for p in site_paths:
if path.find(p) != -1:
return True
return False
new_paths = list(paths)
for p in paths:
if is_site_path(p):
new_paths.remove(p)
return new_paths
sys.path = remove_user_site(sys.path)
rc = pytest.main(plugins=[
collection.CollectionPlugin(test_modules, doctest_modules),
ya,
conftests,
])
if rc == 5:
# don't care about EXIT_NOTESTSCOLLECTED
rc = 0
if rc == 1 and yatest_runner and not listing_mode and not os.environ.get(
FORCE_EXIT_TESTSFAILED_ENV) == '1':
# XXX it's place for future improvements
# Test wrapper should terminate with 0 exit code if there are common test failures
# and report it with trace-file machinery.
# However, there are several case when we don't want to suppress exit_code:
# - listing machinery doesn't use trace-file currently and rely on stdout and exit_code
# - RestartTestException and InfrastructureException required non-zero exit_code to be processes correctly
rc = 0
if profile:
profile.disable()
ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative')
ps.print_stats()
sys.exit(rc)
This module returns the preferred default CA certificate bundle.
If you are packaging Requests, e.g., for a Linux distribution or a managed
environment, you can change the definition of where() to return a separately
packaged CA bundle.
"""
import os.path
import ssl
try:
# load_verify_locations expects PEM cadata to be an ASCII-only unicode object,
# so we discard unicode in comments.
import __res
builtin_cadata = __res.find('/builtin/cacert').decode('ASCII', errors='ignore')
except ImportError:
# Support import from the filesystem for unit2 test runner during elliptics packaging.
builtin_ca = os.path.abspath(os.path.join(os.path.dirname(__file__), 'cacert.pem'))
else:
def builtin_ca():
return None, None, builtin_cadata
def where():
"""Return the preferred certificate bundle."""
# vendored bundle inside Requests
return builtin_ca
if hasattr(ssl, 'SSLContext'):
def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
str_ = lambda s: s
if not isinstance(b'', str):
str_ = lambda s: s.decode('UTF-8')
exceptions = list(no_check)
for key, _ in __res.iter_keys(b'py/no_check_imports/'):
exceptions += str_(__res.find(key)).split()
if exceptions:
exceptions.sort()
print('NO_CHECK_IMPORTS', ' '.join(exceptions))
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
import_times = {}
norm = lambda s: s[:-9] if s.endswith('.__init__') else s
modules = sys.extra_modules | set(extra)
modules = sorted(modules, key=norm)
if py_main:
modules = [py_main] + modules
for module in modules:
if module not in extra and (rx.search(module)
or skip_func and skip_func(module)):
print('SKIP', module)
continue
name = module.rsplit('.', 1)[-1]
if name == '__main__' and 'if __name__ ==' not in importer.get_source(
module):
print('SKIP', module,
'''without "if __name__ == '__main__'" check''')
continue
def print_backtrace_marked(e):
tb_exc = traceback.format_exception(*e)
for item in tb_exc:
for l in item.splitlines():
print('FAIL:', l, file=sys.stderr)
try:
print('TRY', module)
# XXX waiting for py3 to use print(..., flush=True)
sys.stdout.flush()
s = time.time()
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
delay = time.time() - s
import_times[str(module)] = delay
print('OK ', module, '{:.3f}s'.format(delay))
except Exception as e:
print('FAIL:', module, e, file=sys.stderr)
print_backtrace_marked(sys.exc_info())
failed.append('{}: {}'.format(module, e))
except:
e = sys.exc_info()
print('FAIL:', module, e, file=sys.stderr)
print_backtrace_marked(e)
failed.append('{}: {}'.format(module, e))
raise
print("Slowest imports:")
for m, t in sorted(import_times.items(), key=lambda x: x[1],
reverse=True)[:30]:
print(' ', '{:.3f}s'.format(t), m)
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
