def repl(): user_ns = {} py_main = __res.find('PY_MAIN') if py_main: mod_name, func_name = py_main.split(':', 1) try: import importlib mod = importlib.import_module(mod_name) user_ns = mod.__dict__ except: import traceback traceback.print_exc() if py_main and '__main__' not in user_ns: def run(args): if isinstance(args, basestring): import shlex args = shlex.split(args) import sys sys.argv = [sys.argv[0]] + args getattr(mod, func_name)() user_ns['__main__'] = run try: import IPython except ImportError: import code code.interact(local=user_ns) else: IPython.start_ipython(user_ns=user_ns)
def repl(): user_ns = {} py_main = __res.find('PY_MAIN') if py_main: mod_name, func_name = (py_main.split(b':', 1) + [None])[:2] try: import importlib mod = importlib.import_module(mod_name.decode('UTF-8')) user_ns = mod.__dict__ except: import traceback traceback.print_exc() if func_name and '__main__' not in user_ns: def run(args): if isinstance(args, basestring): import shlex args = shlex.split(args) import sys sys.argv = [sys.argv[0]] + args getattr(mod, func_name)() user_ns['__main__'] = run try: import IPython except ModuleNotFoundError: pass else: return IPython.start_ipython(user_ns=user_ns) import code code.interact(local=user_ns)
def main(): skip_names = sys.argv[1:] os.environ['Y_PYTHON_IMPORT_TEST'] = '' # We should initialize Django before importing any applications if os.getenv('DJANGO_SETTINGS_MODULE'): try: import django except ImportError: pass else: django.setup() py_main = __res.find('PY_MAIN') if py_main: py_main_module = py_main.split(b':', 1)[0].decode('UTF-8') else: py_main_module = None try: check_imports(no_check=skip_names, py_main=py_main_module) except: sys.exit(1)
def builtin_cadata(): global _builtin_cadata if _builtin_cadata is None: import __res data = __res.find(b'/builtin/cacert') # load_verify_locations expects PEM cadata to be an ASCII-only unicode # object, so we discard unicode in comments. _builtin_cadata = data.decode('ASCII', errors='ignore') return _builtin_cadata
def check_imports(no_check=(), extra=(), skip_func=None): """ tests all bundled modules are importable just add "PEERDIR(library/python/import_test)" to your CMakeLists.txt and "from import_test import test_imports" to your python test source file. """ str_ = lambda s: s if not isinstance(b'', str): str_ = lambda s: s.decode('UTF-8') exceptions = list(no_check) for key, _ in __res.iter_keys(b'py/no_check_imports/'): exceptions += str_(__res.find(key)).split() if exceptions: exceptions.sort() print('NO_CHECK_IMPORTS', ' '.join(exceptions)) patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions] rx = re.compile('^({})$'.format('|'.join(patterns))) failed = [] norm = lambda s: s[:-9] if s.endswith('.__init__') else s for module in sorted(sys.extra_modules | set(extra), key=norm): if module not in extra and (rx.search(module) or skip_func and skip_func(module)): print('SKIP', module) continue name = module.rsplit('.', 1)[-1] if name == '__main__' and 'if __name__ ==' not in importer.get_source( module): print('SKIP', module, '''without "if __name__ == '__main__'" check''') continue try: print('TRY', module) if module == '__main__': importer.load_module('__main__', '__main__py') elif module.endswith('.__init__'): __import__(module[:-len('.__init__')]) else: __import__(module) print('OK ', module) except Exception as e: print('FAIL', module, e, file=sys.stderr) traceback.print_exception(*sys.exc_info()) failed.append('{}: {}'.format(module, e)) if failed: raise ImportError('modules not imported:\n' + '\n'.join(failed))
def open_resource(name): """Open a resource from the zoneinfo for reading. Uses compiled-in timezone files """ import __res from cStringIO import StringIO res_name = '/pytz_data/zoneinfo/' + name.lstrip('/') res_buf = __res.find(res_name) if res_buf is None: raise IOError("No such resource: " + res_name) return StringIO(res_buf)
def repl(): user_ns = {} py_main = __res.find('PY_MAIN') if py_main: py_main_split = py_main.split(':', 1) if len(py_main_split) == 2: mod_name, func_name = py_main_split else: mod_name, func_name = py_main_split[0], 'main' if not mod_name: mod_name = 'library.python.runtime.entry_points' try: import importlib mod = importlib.import_module(mod_name) user_ns = mod.__dict__ except: import traceback traceback.print_exc() if '__main__' not in user_ns: def run(args): if isinstance(args, basestring): import shlex args = shlex.split(args) import sys sys.argv = [sys.argv[0]] + args getattr(mod, func_name)() user_ns['__main__'] = run else: try: mod = __res.importer.load_module('__main__', fix_name='__main_real') user_ns = mod.__dict__ except ImportError: pass try: import IPython except ImportError: import code code.interact(local=user_ns) else: IPython.start_ipython(user_ns=user_ns)
def _file_contents(file_name): if getattr(sys, 'is_standalone_binary', False): import __res for prefix in ['/coredump_filter_data/', '']: res_name = prefix + file_name data = __res.find(res_name) if data is not None: return data raise IOError("Failed to find resource: " + file_name) else: if not os.path.exists(file_name): file_name = os.path.join(MY_PATH, file_name) with open(file_name) as f: return f.read()
def iteritems(prefix='', strip_prefix=False): for key in iterkeys(prefix=prefix): value = find(key) if strip_prefix: key = key[len(prefix):] yield key, value
def run_constructors(): for key, module_name in __res.iter_keys(b'py/constructors/'): import importlib module = importlib.import_module(module_name.decode()) init_func = getattr(module, __res.find(key).decode()) init_func()
def updatecache(filename, module_globals=None): """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list.""" if filename in cache: del cache[filename] if not filename or (filename.startswith('<') and filename.endswith('>')): return [] if not os.path.isabs(filename): # mod.__loader__.get_source works, but when mod fails to import # mod.__loader__ and other mod globals are already None. import __res key = __res.importer.file_source(filename) if key: data = __res.find(key) assert data is not None, filename lines = [line + '\n' for line in data.splitlines()] cache[filename] = (len(data), None, lines, filename) return cache[filename][2] fullname = filename try: stat = os.stat(fullname) except OSError: basename = filename # Try for a __loader__, if available if module_globals and '__loader__' in module_globals: name = module_globals.get('__name__') loader = module_globals['__loader__'] get_source = getattr(loader, 'get_source', None) if name and get_source: try: data = get_source(name) except (ImportError, IOError): pass else: if data is None: # No luck, the PEP302 loader cannot find the source # for this module. return [] cache[filename] = ( len(data), None, [line+'\n' for line in data.splitlines()], fullname ) return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): return [] for dirname in sys.path: # When using imputil, sys.path may contain things other than # strings; ignore them when it happens. try: fullname = os.path.join(dirname, basename) except (TypeError, AttributeError): # Not sufficiently string-like to do anything useful with. continue try: stat = os.stat(fullname) break except os.error: pass else: return [] try: with open(fullname, 'rU') as fp: lines = fp.readlines() except IOError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines
def updatecache(filename, module_globals=None): """Update a cache entry and return its list of lines. If something's wrong, print a message, discard the cache entry, and return an empty list.""" if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return [] if not os.path.isabs(filename): # Do not read builtin code from the filesystem. import __res key = __res.importer.file_source(filename) if key: data = __res.find(key) assert data is not None, filename data = data.decode('UTF-8') lines = [line + '\n' for line in data.splitlines()] cache[filename] = (len(data), None, lines, filename) return cache[filename][2] fullname = filename try: stat = os.stat(fullname) except OSError: basename = filename # Realise a lazy loader based lookup if there is one # otherwise try to lookup right now. if lazycache(filename, module_globals): try: data = cache[filename][0]() except (ImportError, OSError): pass else: if data is None: # No luck, the PEP302 loader cannot find the source # for this module. return [] cache[filename] = (len(data), None, [line + '\n' for line in data.splitlines()], fullname) return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. if os.path.isabs(filename): return [] for dirname in sys.path: try: fullname = os.path.join(dirname, basename) except (TypeError, AttributeError): # Not sufficiently string-like to do anything useful with. continue try: stat = os.stat(fullname) break except OSError: pass else: return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except OSError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines
def itervalues(prefix=b''): for key in iterkeys(prefix=prefix): value = find(key) yield value
def main(): import library.python.pytest.context as context context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() profile = None if '--profile-pytest' in sys.argv: sys.argv.remove('--profile-pytest') import pstats import cProfile profile = cProfile.Profile() profile.enable() import pytest import library.python.pytest.plugins.collection as collection import library.python.pytest.plugins.ya as ya import library.python.pytest.plugins.conftests as conftests import _pytest.assertion from _pytest.monkeypatch import MonkeyPatch from . import rewrite m = MonkeyPatch() m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook", rewrite.AssertionRewritingHook) prefix = '__tests__.' test_modules = [ name[len(prefix):] for name in sys.extra_modules if name.startswith(prefix) and not name.endswith('.conftest') ] doctest_packages = (__res.find("PY_DOCTEST_PACKAGES") or "").split() def is_doctest_module(name): for package in doctest_packages: if name == package or name.startswith(package + "."): return True return False doctest_modules = [ name for name in sys.extra_modules if is_doctest_module(name) ] def remove_user_site(paths): site_paths = ('site-packages', 'site-python') def is_site_path(path): for p in site_paths: if path.find(p) != -1: return True return False new_paths = list(paths) for p in paths: if is_site_path(p): new_paths.remove(p) return new_paths sys.path = remove_user_site(sys.path) rc = pytest.main(plugins=[ collection.CollectionPlugin(test_modules, doctest_modules), ya, conftests, ]) if rc == 5: # don't care about EXIT_NOTESTSCOLLECTED rc = 0 if profile: profile.disable() ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative') ps.print_stats() sys.exit(rc)
def main(): import library.python.pytest.context as context context.Ctx["YA_PYTEST_START_TIMESTAMP"] = time.time() profile = None if '--profile-pytest' in sys.argv: sys.argv.remove('--profile-pytest') import pstats import cProfile profile = cProfile.Profile() profile.enable() # Reset influencing env. vars # For more info see library/python/testing/yatest_common/yatest/common/errors.py if FORCE_EXIT_TESTSFAILED_ENV in os.environ: del os.environ[FORCE_EXIT_TESTSFAILED_ENV] if "Y_PYTHON_CLEAR_ENTRY_POINT" in os.environ: if "Y_PYTHON_ENTRY_POINT" in os.environ: del os.environ["Y_PYTHON_ENTRY_POINT"] del os.environ["Y_PYTHON_CLEAR_ENTRY_POINT"] listing_mode = '--collect-only' in sys.argv yatest_runner = os.environ.get('YA_TEST_RUNNER') == '1' import pytest import library.python.pytest.plugins.collection as collection import library.python.pytest.plugins.ya as ya import library.python.pytest.plugins.conftests as conftests import _pytest.assertion from _pytest.monkeypatch import MonkeyPatch from . import rewrite m = MonkeyPatch() m.setattr(_pytest.assertion.rewrite, "AssertionRewritingHook", rewrite.AssertionRewritingHook) prefix = '__tests__.' test_modules = [ name[len(prefix):] for name in sys.extra_modules if name.startswith(prefix) and not name.endswith('.conftest') ] doctest_packages = __res.find("PY_DOCTEST_PACKAGES") or "" if isinstance(doctest_packages, bytes): doctest_packages = doctest_packages.decode('utf-8') doctest_packages = doctest_packages.split() def is_doctest_module(name): for package in doctest_packages: if name == package or name.startswith(str(package) + "."): return True return False doctest_modules = [ name for name in sys.extra_modules if is_doctest_module(name) ] def remove_user_site(paths): site_paths = ('site-packages', 'site-python') def is_site_path(path): for p in site_paths: if path.find(p) != -1: return True return False new_paths = list(paths) for p in paths: if is_site_path(p): new_paths.remove(p) return new_paths sys.path = remove_user_site(sys.path) rc = pytest.main(plugins=[ collection.CollectionPlugin(test_modules, doctest_modules), ya, conftests, ]) if rc == 5: # don't care about EXIT_NOTESTSCOLLECTED rc = 0 if rc == 1 and yatest_runner and not listing_mode and not os.environ.get( FORCE_EXIT_TESTSFAILED_ENV) == '1': # XXX it's place for future improvements # Test wrapper should terminate with 0 exit code if there are common test failures # and report it with trace-file machinery. # However, there are several case when we don't want to suppress exit_code: # - listing machinery doesn't use trace-file currently and rely on stdout and exit_code # - RestartTestException and InfrastructureException required non-zero exit_code to be processes correctly rc = 0 if profile: profile.disable() ps = pstats.Stats(profile, stream=sys.stderr).sort_stats('cumulative') ps.print_stats() sys.exit(rc)
This module returns the preferred default CA certificate bundle. If you are packaging Requests, e.g., for a Linux distribution or a managed environment, you can change the definition of where() to return a separately packaged CA bundle. """ import os.path import ssl try: # load_verify_locations expects PEM cadata to be an ASCII-only unicode object, # so we discard unicode in comments. import __res builtin_cadata = __res.find('/builtin/cacert').decode('ASCII', errors='ignore') except ImportError: # Support import from the filesystem for unit2 test runner during elliptics packaging. builtin_ca = os.path.abspath(os.path.join(os.path.dirname(__file__), 'cacert.pem')) else: def builtin_ca(): return None, None, builtin_cadata def where(): """Return the preferred certificate bundle.""" # vendored bundle inside Requests return builtin_ca if hasattr(ssl, 'SSLContext'):
def check_imports(no_check=(), extra=(), skip_func=None, py_main=None): """ tests all bundled modules are importable just add "PEERDIR(library/python/import_test)" to your CMakeLists.txt and "from import_test import test_imports" to your python test source file. """ str_ = lambda s: s if not isinstance(b'', str): str_ = lambda s: s.decode('UTF-8') exceptions = list(no_check) for key, _ in __res.iter_keys(b'py/no_check_imports/'): exceptions += str_(__res.find(key)).split() if exceptions: exceptions.sort() print('NO_CHECK_IMPORTS', ' '.join(exceptions)) patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions] rx = re.compile('^({})$'.format('|'.join(patterns))) failed = [] import_times = {} norm = lambda s: s[:-9] if s.endswith('.__init__') else s modules = sys.extra_modules | set(extra) modules = sorted(modules, key=norm) if py_main: modules = [py_main] + modules for module in modules: if module not in extra and (rx.search(module) or skip_func and skip_func(module)): print('SKIP', module) continue name = module.rsplit('.', 1)[-1] if name == '__main__' and 'if __name__ ==' not in importer.get_source( module): print('SKIP', module, '''without "if __name__ == '__main__'" check''') continue def print_backtrace_marked(e): tb_exc = traceback.format_exception(*e) for item in tb_exc: for l in item.splitlines(): print('FAIL:', l, file=sys.stderr) try: print('TRY', module) # XXX waiting for py3 to use print(..., flush=True) sys.stdout.flush() s = time.time() if module == '__main__': importer.load_module('__main__', '__main__py') elif module.endswith('.__init__'): __import__(module[:-len('.__init__')]) else: __import__(module) delay = time.time() - s import_times[str(module)] = delay print('OK ', module, '{:.3f}s'.format(delay)) except Exception as e: print('FAIL:', module, e, file=sys.stderr) print_backtrace_marked(sys.exc_info()) failed.append('{}: {}'.format(module, e)) except: e = sys.exc_info() print('FAIL:', module, e, file=sys.stderr) print_backtrace_marked(e) failed.append('{}: {}'.format(module, e)) raise print("Slowest imports:") for m, t in sorted(import_times.items(), key=lambda x: x[1], reverse=True)[:30]: print(' ', '{:.3f}s'.format(t), m) if failed: raise ImportError('modules not imported:\n' + '\n'.join(failed))