def _rewrite_test(config, name):
"""Try to read and rewrite *fn* and return the code object."""
state = config._assertstate
source = importer.get_source(name)
if source is None:
return None
path = importer.get_filename(name)
try:
tree = ast.parse(source, filename=path)
except SyntaxError:
# Let this pop up again in the real import.
state.trace("failed to parse: %r" % (path, ))
return None
rewrite.rewrite_asserts(tree, path, config)
try:
co = compile(tree, path, "exec", dont_inherit=True)
except SyntaxError:
# It's possible that this error is from some bug in the
# assertion rewriting, but I don't know of a fast way to tell.
state.trace("failed to compile: %r" % (path, ))
return None
return co
def check_imports(no_check=(), extra=(), skip_func=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
str_ = lambda s: s
if not isinstance(b'', str):
str_ = lambda s: s.decode('UTF-8')
exceptions = list(no_check)
for key, _ in __res.iter_keys(b'py/no_check_imports/'):
exceptions += str_(__res.find(key)).split()
if exceptions:
exceptions.sort()
print('NO_CHECK_IMPORTS', ' '.join(exceptions))
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
norm = lambda s: s[:-9] if s.endswith('.__init__') else s
for module in sorted(sys.extra_modules | set(extra), key=norm):
if module not in extra and (rx.search(module)
or skip_func and skip_func(module)):
print('SKIP', module)
continue
name = module.rsplit('.', 1)[-1]
if name == '__main__' and 'if __name__ ==' not in importer.get_source(
module):
print('SKIP', module,
'''without "if __name__ == '__main__'" check''')
continue
try:
print('TRY', module)
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
print('OK ', module)
except Exception as e:
print('FAIL', module, e, file=sys.stderr)
traceback.print_exception(*sys.exc_info())
failed.append('{}: {}'.format(module, e))
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
def check_imports(no_check=None, extra=[], skip_func=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
exceptions = [
'__yt_entry_point__',
'boto.*',
'click._winconsole',
'common.*', # sandbox.common
'flask.ext.__init__',
'future.backports.email.policy', # email backport is incomplete in v0.16.0.
'future.moves.dbm.ndbm',
'gensim.models.lda_worker',
'gensim.models.lda_dispatcher',
'gensim.models.lsi_dispatcher',
'gensim.models.lsi_worker',
'gensim.similarities.index',
'kernel.*', # skynet/kernel
'IPython.*',
'lxml.cssselect',
'lxml.html.ElementSoup',
'lxml.html.diff',
'lxml.html._diffcommand',
'lxml.html._html5builder',
'lxml.html.html5parser',
'lxml.html.soupparser',
'lxml.html.usedoctest',
'lxml.isoschematron.__init__',
'lxml.usedoctest',
'py._code._assertionnew',
'py._code._assertionold',
'rbtools.clients.tests',
'requests.__init__',
'requests.packages.chardet.chardetect',
'requests.packages.urllib3.contrib.ntlmpool',
'setuptools.*',
'_pytest.*',
'__tests__.*', # all test modules get imported when tests are run
"yt.packages.*",
"yt.wrapper.cypress_fuse",
"tornado.platform.*",
"tornado.curl_httpclient",
"google.protobuf.internal.cpp_message",
"google.protobuf.pyext.cpp_message",
"numpy.distutils.*",
"numpy.core.setup",
"numpy.core.cversions",
"numpy.lib.__init__",
"numpy.doc.*",
"numpy.testing.__init__",
"numpy.ma.version",
"numpy.matrixlib.__init__",
"numpy.testing.noseclasses",
"numpy.__init__",
"numpy.core.code_generators.generate_numpy_api",
"numpy.doc.basics",
"numpy.doc.broadcasting",
"scipy.misc.__init__", # XXX: fixme
"kazoo.handlers.*",
"psutil._pssunos",
"psutil._psbsd",
"psutil._psosx",
"psutil._pswindows",
"psutil._psutil_common",
"psutil._psutil_bsd",
"psutil._psutil_osx",
"psutil._psutil_sunos",
"psutil._psutil_windows",
"PIL.ImageCms",
"PIL.ImageGL",
"PIL.ImageGrab",
"PIL.ImageQt",
"PIL.ImageTk",
"coloredlogs.cli",
"dateutil.tzwin",
"dateutil.tz.win",
"matplotlib.backends.*",
"matplotlib.sphinxext.*",
"matplotlib.testing.*",
"mpl_toolkits.*",
"mwlib.*",
"networkx.algorithms.connectivity.__init__",
"networkx.algorithms.flow.__init__",
"networkx.testing.__init__",
"nile.drivers.yql.*",
"nile.style.jupyter_monitor",
"pandas.io.auth",
"pandas.io.data",
"pandas.io.ga",
"pandas.io.s3",
"pandas.io.wb",
"pandas.rpy.base",
"pandas.rpy.common",
"pandas.rpy.vars",
"pandas.util.clipboard",
"parsel.unified",
"ptpython.contrib.asyncssh_repl",
"ptpython.ipython",
"prompt_toolkit.clipboard.pyperclip",
"prompt_toolkit.eventloop.asyncio_posix",
"prompt_toolkit.eventloop.asyncio_win32",
"prompt_toolkit.eventloop.win32",
"prompt_toolkit.terminal.conemu_output",
"prompt_toolkit.win32_types",
"prompt_toolkit.terminal.win32_input",
"prompt_toolkit.terminal.win32_output",
"backports.__init__",
"pygments.sphinxext",
"raven.contrib.*",
"raven.handlers.logbook",
"raven.utils.testutils",
"sklearn.utils.*",
"subvertpy.ra_svn", # can only be imported after subvertpy.ra
"superfcgi.*",
"thrift.TSCons",
"thrift.TTornado",
"thrift.transport.*",
"twisted.*",
"uwsgidecorators",
"watchdog.*",
"werkzeug.*",
"ws4py.*",
'wtforms.ext.django.*',
"services.lfm.*",
"sqlalchemy.testing",
"gevent.win32util",
"library.python.ctypes.__init__",
"celery.events.cursesmon",
"billiard.popen_forkserver",
"billiard.forkserver",
"celery.contrib.sphinx",
"flask_wtf.i18n",
"playhouse.apsw_ext",
"botocore.vendored.requests.packages.urllib3.contrib.pyopenssl",
"cssutils._fetchgae",
"catboost.widget.*",
"kubiki.geobase",
] + list(no_check or [])
if sys.version_info.major == 3:
exceptions += [
"antigravity",
"lzma",
"dbm.ndbm",
"tkinter",
"msvcrt",
"msilib.*",
"_msi",
"winreg",
"asyncio.test_utils",
"dbm.gnu",
"multiprocessing.popen_spawn_win32",
"encodings.mbcs",
"turtle",
"ctypes.wintypes",
"asyncio.windows_utils",
"distutils.msvc9compiler",
"distutils._msvccompiler",
"urllib3.packages.ordered_dict",
"encodings.oem",
"crypt",
"asyncio.windows_events",
"encodings.cp65001",
"curses.*",
"distutils.command.bdist_msi",
"yaml.cyaml",
"vh.ext.nirvana.nirvana_api_bridge",
]
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
for module in sys.extra_modules:
if rx.search(module):
continue
if skip_func and skip_func(module):
continue
if module == '__main__' and 'if __name__ ==' not in importer.get_source(
module):
print('SKIP:', module,
'''without "if __name__ == '__main__'" check''')
continue
try:
print('TRY:', module)
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
print('OK:', module)
except Exception as e:
print('FAIL:', module, e, file=sys.stderr)
traceback.print_exception(*sys.exc_info())
failed.append('{}: {}'.format(module, e))
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
def get_source(self, name):
return importer.get_source(name)
def check_imports(no_check=(), extra=(), skip_func=None, py_main=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
str_ = lambda s: s
if not isinstance(b'', str):
str_ = lambda s: s.decode('UTF-8')
exceptions = list(no_check)
for key, _ in __res.iter_keys(b'py/no_check_imports/'):
exceptions += str_(__res.find(key)).split()
if exceptions:
exceptions.sort()
print('NO_CHECK_IMPORTS', ' '.join(exceptions))
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
import_times = {}
norm = lambda s: s[:-9] if s.endswith('.__init__') else s
modules = sys.extra_modules | set(extra)
modules = sorted(modules, key=norm)
if py_main:
modules = [py_main] + modules
for module in modules:
if module not in extra and (rx.search(module)
or skip_func and skip_func(module)):
print('SKIP', module)
continue
name = module.rsplit('.', 1)[-1]
if name == '__main__' and 'if __name__ ==' not in importer.get_source(
module):
print('SKIP', module,
'''without "if __name__ == '__main__'" check''')
continue
def print_backtrace_marked(e):
tb_exc = traceback.format_exception(*e)
for item in tb_exc:
for l in item.splitlines():
print('FAIL:', l, file=sys.stderr)
try:
print('TRY', module)
# XXX waiting for py3 to use print(..., flush=True)
sys.stdout.flush()
s = time.time()
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
delay = time.time() - s
import_times[str(module)] = delay
print('OK ', module, '{:.3f}s'.format(delay))
except Exception as e:
print('FAIL:', module, e, file=sys.stderr)
print_backtrace_marked(sys.exc_info())
failed.append('{}: {}'.format(module, e))
except:
e = sys.exc_info()
print('FAIL:', module, e, file=sys.stderr)
print_backtrace_marked(e)
failed.append('{}: {}'.format(module, e))
raise
print("Slowest imports:")
for m, t in sorted(import_times.items(), key=lambda x: x[1],
reverse=True)[:30]:
print(' ', '{:.3f}s'.format(t), m)
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
def check_imports(no_check=None, extra=[], skip_func=None):
"""
tests all bundled modules are importable
just add
"PEERDIR(library/python/import_test)" to your CMakeLists.txt and
"from import_test import test_imports" to your python test source file.
"""
exceptions = [
'__yt_entry_point__',
'boto.*',
'click._winconsole',
'common.*', # sandbox.common
'flask.ext.__init__',
'future.backports.email.policy', # email backport is incomplete in v0.16.0.
'gensim.models.lda_worker',
'gensim.models.lda_dispatcher',
'gensim.models.lsi_dispatcher',
'gensim.models.lsi_worker',
'gensim.similarities.index',
'kernel.*', # skynet/kernel
'IPython.*',
'lxml.cssselect',
'lxml.html.ElementSoup',
'lxml.html.diff',
'lxml.html._diffcommand',
'lxml.html._html5builder',
'lxml.html.html5parser',
'lxml.html.soupparser',
'lxml.html.usedoctest',
'lxml.isoschematron.__init__',
'lxml.usedoctest',
'py._code._assertionnew',
'py._code._assertionold',
'rbtools.clients.tests',
'requests.__init__',
'requests.packages.chardet.chardetect',
'requests.packages.urllib3.contrib.ntlmpool',
'setuptools.*',
'_pytest.assertion.newinterpret',
'__tests__.*', # all test modules get imported when tests are run
"yt.packages.*",
"tornado.platform.*",
"tornado.curl_httpclient",
"google.protobuf.internal.cpp_message",
"google.protobuf.pyext.cpp_message",
"numpy.distutils.*",
"numpy.core.setup",
"numpy.core.cversions",
"numpy.lib.__init__",
"numpy.doc.*",
"numpy.testing.__init__",
"numpy.ma.version",
"numpy.matrixlib.__init__",
"numpy.testing.noseclasses",
"numpy.__init__",
"numpy.core.code_generators.generate_numpy_api",
"numpy.doc.basics",
"numpy.doc.broadcasting",
"scipy.misc.__init__", # XXX: fixme
"kazoo.handlers.*",
"psutil._pssunos",
"psutil._psbsd",
"psutil._psosx",
"psutil._pswindows",
"psutil._psutil_common",
"psutil._psutil_bsd",
"psutil._psutil_osx",
"psutil._psutil_sunos",
"psutil._psutil_windows",
"PIL.ImageCms",
"PIL.ImageGL",
"PIL.ImageGrab",
"PIL.ImageQt",
"PIL.ImageTk",
"coloredlogs.cli",
"dateutil.tzwin",
"matplotlib.backends.*",
"matplotlib.sphinxext.*",
"matplotlib.testing.*",
"mpl_toolkits.*",
"networkx.algorithms.connectivity.__init__",
"networkx.algorithms.flow.__init__",
"networkx.testing.__init__",
"nile.*",
"pandas.io.auth",
"pandas.io.data",
"pandas.io.ga",
"pandas.io.s3",
"pandas.io.wb",
"pandas.rpy.base",
"pandas.rpy.vars",
"pandas.util.clipboard",
"parsel.unified",
"ptpython.contrib.asyncssh_repl",
"ptpython.ipython",
"prompt_toolkit.clipboard.pyperclip",
"prompt_toolkit.eventloop.asyncio_posix",
"prompt_toolkit.eventloop.asyncio_win32",
"prompt_toolkit.eventloop.win32",
"prompt_toolkit.terminal.conemu_output",
"prompt_toolkit.win32_types",
"prompt_toolkit.terminal.win32_input",
"prompt_toolkit.terminal.win32_output",
"backports.__init__",
"pygments.sphinxext",
"qb2.doc.extractor_graph",
"qb2.doc.regex",
"qb2.doc.resources",
"raven.contrib.*",
"raven.handlers.logbook",
"raven.utils.testutils",
"subvertpy.ra_svn", # can only be imported after subvertpy.ra
"superfcgi.*",
"thrift.TSCons",
"thrift.TTornado",
"thrift.transport.*",
"twisted.*",
"uwsgidecorators",
"watchdog.*",
"werkzeug.*",
"ws4py.*",
'wtforms.ext.django.*',
"services.lfm.*",
"sqlalchemy.testing",
"gevent.win32util",
"library.python.ctypes.__init__",
] + list(no_check or [])
patterns = [re.escape(s).replace(r'\*', r'.*') for s in exceptions]
rx = re.compile('^({})$'.format('|'.join(patterns)))
failed = []
for module in sys.extra_modules:
if rx.search(module):
continue
if skip_func and skip_func(module):
continue
if module == '__main__' and 'if __name__ ==' not in importer.get_source(module):
print 'SKIP:', module, '''without "if __name__ == '__main__'" check'''
continue
try:
print 'TRY:', module
if module == '__main__':
importer.load_module('__main__', '__main__py')
elif module.endswith('.__init__'):
__import__(module[:-len('.__init__')])
else:
__import__(module)
print 'OK:', module
except Exception as e:
print >>sys.stderr, 'FAIL:', module, e
traceback.print_exception(*sys.exc_info())
failed.append('{}: {}'.format(module, e))
if failed:
raise ImportError('modules not imported:\n' + '\n'.join(failed))
