def create_cffi_import_libraries(pypy_c, options, basedir):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
ignore_errors=True)
# be sure pip, setuptools are installed in a fresh pypy
# allows proper functioning of cffi on win32 with newer vc compilers
# XXX move this to a build slave step?
status, stdout, stderr = run_subprocess(str(pypy_c), ['-c', 'import setuptools'])
if status != 0:
status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip'])
failures = []
for key, module in sorted(cffi_build_scripts.items()):
if module is None or getattr(options, 'no_' + key, False):
continue
if module.endswith('.py'):
args = [module]
cwd = str(basedir.join('lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
print('*', ' '.join(args), file=sys.stderr)
try:
status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
except:
import traceback;traceback.print_exc()
failures.append((key, module))
return failures
def _build_dependency(name, patches=[]):
import shutil
import subprocess
from rpython.tool.runsubprocess import run_subprocess
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
try:
url, dgst, build_cmds = cffi_dependencies[name]
except KeyError:
return 0, None, None
archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')
if not os.path.isdir(archive_dir):
os.makedirs(archive_dir)
archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])
# next, fetch the archive to disk, if needed
if not os.path.exists(archive) or _sha256(archive) != dgst:
print('fetching archive', url, file=sys.stderr)
urlretrieve(url, archive)
shutil.rmtree(deps_destdir, ignore_errors=True)
os.makedirs(deps_destdir)
# extract the into our destination directory
print('unpacking archive', archive, file=sys.stderr)
_unpack_tarfile(archive, deps_destdir)
sources = os.path.join(deps_destdir, os.path.basename(archive)[:-7])
# apply any patches
if patches:
for patch in patches:
print('applying patch', patch, file=sys.stderr)
status, stdout, stderr = run_subprocess(
'/usr/bin/patch',
['-p1', '-i', patch],
cwd=sources,
)
if status != 0:
return status, stdout, stderr
for args in build_cmds:
print('running', ' '.join(args), 'in', sources, file=sys.stderr)
status, stdout, stderr = run_subprocess(
args[0],
args[1:],
cwd=sources,
)
if status != 0:
break
return status, stdout, stderr
def test_cat_fail():
if not os.path.exists("/bin/cat"):
py.test.skip("there is no /bin/cat")
returncode, stdout, stderr = run_subprocess("/bin/cat", "no/such/filename")
assert returncode == 1
assert stdout == ""
assert "no/such/filename" in stderr
def test_false():
if not os.path.exists("/bin/false"):
py.test.skip("there is no /bin/false")
returncode, stdout, stderr = run_subprocess("/bin/false", [])
assert returncode == 1
assert stdout == ""
assert stderr == ""
def create_cffi_import_libraries(pypy_c, options, basedir):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
ignore_errors=True)
failures = []
for key, module in sorted(cffi_build_scripts.items()):
if module is None or getattr(options, 'no_' + key, False):
continue
if module.endswith('.py'):
args = [module]
cwd = str(basedir.join('lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
print('*', ' '.join(args), file=sys.stderr)
try:
status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
except:
import traceback;traceback.print_exc()
failures.append((key, module))
return failures
def test_echo():
if not os.path.exists("/bin/echo"):
py.test.skip("there is no /bin/echo")
returncode, stdout, stderr = run_subprocess("/bin/echo", "FooBar")
assert returncode == 0
assert stdout == "FooBar\n"
assert stderr == ""
def task_build_cffi_imports(self):
''' Use cffi to compile cffi interfaces to modules'''
filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
status, out, err = run_subprocess(str(driver.compute_exe_name()),
[filename])
sys.stdout.write(out)
sys.stderr.write(err)
def test_echo():
if not os.path.exists('/bin/echo'):
py.test.skip("there is no /bin/echo")
returncode, stdout, stderr = run_subprocess('/bin/echo', 'FooBar')
assert returncode == 0
assert stdout == 'FooBar\n'
assert stderr == ''
def test_false():
if not os.path.exists('/bin/false'):
py.test.skip("there is no /bin/false")
returncode, stdout, stderr = run_subprocess('/bin/false', [])
assert returncode == 1
assert stdout == ''
assert stderr == ''
def test_cat_fail():
if not os.path.exists('/bin/cat'):
py.test.skip("there is no /bin/cat")
returncode, stdout, stderr = run_subprocess('/bin/cat', 'no/such/filename')
assert returncode == 1
assert stdout == ''
assert 'no/such/filename' in stderr
def task_build_cffi_imports(self):
''' Use cffi to compile cffi interfaces to modules'''
filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py')
status, out, err = run_subprocess(str(driver.compute_exe_name()),
[filename])
sys.stdout.write(out)
sys.stderr.write(err)
def create_cffi_import_libraries(pypy_c, options, basedir):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(basedir.join('lib_pypy', '__pycache__')),
ignore_errors=True)
failures = []
for key, module in sorted(cffi_build_scripts.items()):
if module is None or getattr(options, 'no_' + key, False):
continue
if module.endswith('.py'):
args = [module]
cwd = str(basedir.join('lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
print('*', ' '.join(args), file=sys.stderr)
try:
status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
except:
import traceback
traceback.print_exc()
failures.append((key, module))
return failures
def is_pypy(self):
if self._is_pypy is not None:
return self._is_pypy
CODE = "import sys; print('__pypy__' in sys.builtin_module_names)"
res, stdout, stderr = run_subprocess(self.path, ["-c", CODE])
if res != 0:
raise ValueError("Invalid Python interpreter")
print stdout
is_pypy = stdout.strip() == 'True'
self._is_pypy = is_pypy
return is_pypy
def task_build_cffi_imports(self):
''' Use cffi to compile cffi interfaces to modules'''
filename = join(pypydir, '..', 'lib_pypy', 'pypy_tools',
'build_cffi_imports.py')
if sys.platform == 'darwin':
argv = [filename, '--embed-dependencies']
else:
argv = [filename,]
status, out, err = run_subprocess(str(driver.compute_exe_name()),
argv)
sys.stdout.write(out)
sys.stderr.write(err)
def get_recent_cpython_executable():
if sys.platform == 'win32':
python = sys.executable.replace('\\', '/')
else:
python = sys.executable
# Is there a command 'python' that runs python 2.5-2.7?
# If there is, then we can use it instead of sys.executable
returncode, stdout, stderr = runsubprocess.run_subprocess("python", "-V")
if _CPYTHON_RE.match(stdout) or _CPYTHON_RE.match(stderr):
python = 'python'
return python
def get_recent_cpython_executable():
if sys.platform == 'win32':
python = sys.executable.replace('\\', '/')
else:
python = sys.executable
# Is there a command 'python' that runs python 2.5-2.7?
# If there is, then we can use it instead of sys.executable
returncode, stdout, stderr = runsubprocess.run_subprocess(
"python", "-V")
if _CPYTHON_RE.match(stdout) or _CPYTHON_RE.match(stderr):
python = 'python'
return python
def task_build_cffi_imports(self):
''' Use cffi to compile cffi interfaces to modules'''
filename = join(pypydir, '..', 'lib_pypy', 'pypy_tools',
'build_cffi_imports.py')
if sys.platform in ('darwin', 'linux', 'linux2'):
argv = [filename, '--embed-dependencies']
else:
argv = [
filename,
]
exe_name = py.path.local(driver.c_entryp)
status, out, err = run_subprocess(str(exe_name), argv)
sys.stdout.write(out)
sys.stderr.write(err)
def run_with_python(python_, target_, usemodules, **definitions):
if python_ is None:
py.test.skip("Cannot find the default python3 interpreter to run with -A")
# we assume that the source of target_ is in utf-8. Unfortunately, we don't
# have any easy/standard way to determine from here the original encoding
# of the source file
helpers = r"""# -*- encoding: utf-8 -*-
if 1:
import sys
sys.path.append(%r)
%s
def skip(message):
print(message)
raise SystemExit(0)
__builtins__.skip = skip
__builtins__.py3k_skip = skip
class ExceptionWrapper:
pass
def raises(exc, func, *args, **kwargs):
import os
try:
if isinstance(func, str):
if func.startswith((' ', os.linesep)):
# it's probably an indented block, so we prefix if True:
# to avoid SyntaxError
func = "if True:\n" + func
frame = sys._getframe(1)
exec(func, frame.f_globals, frame.f_locals)
else:
func(*args, **kwargs)
except exc as e:
res = ExceptionWrapper()
res.value = e
return res
else:
raise AssertionError("DID NOT RAISE")
__builtins__.raises = raises
class Test:
pass
self = Test()
"""
defs = []
for symbol, value in sorted(definitions.items()):
if isinstance(value, tuple) and isinstance(value[0], py.code.Source):
code, args = value
defs.append(str(code))
arg_repr = []
for arg in args:
if isinstance(arg, types.FunctionType):
arg_repr.append(arg.__name__)
elif isinstance(arg, types.MethodType):
arg_repr.append(arg.__name__)
else:
arg_repr.append(py3k_repr(arg))
args = ', '.join(arg_repr)
defs.append("self.%s = anonymous(%s)\n" % (symbol, args))
elif isinstance(value, types.MethodType):
# "def w_method(self)"
code = py.code.Code(value)
defs.append(str(code.source()))
defs.append("type(self).%s = %s\n" % (symbol, value.__name__))
elif isinstance(value, types.ModuleType):
name = value.__name__
defs.append("import %s; self.%s = %s\n" % (name, symbol, name))
elif isinstance(value, (str, unicode, int, long, float, list, tuple,
dict)) or value is None:
defs.append("self.%s = %s\n" % (symbol, py3k_repr(value)))
check_usemodules = ''
if usemodules:
usemodules = [str(RENAMED_USEMODULES.get(name, name))
for name in usemodules]
check_usemodules = """\
missing = set(%r).difference(sys.builtin_module_names)
if missing:
if not hasattr(sys, 'pypy_version_info'):
# They may be extension modules on CPython
name = None
for name in missing.copy():
try:
__import__(name)
except ImportError:
pass
else:
missing.remove(name)
del name
if missing:
sys.exit(81)
del missing""" % usemodules
source = list(py.code.Source(target_))
while source[0].startswith(('@py.test.mark.', '@pytest.mark.')):
source.pop(0)
source = source[1:]
pyfile = udir.join('src.py')
if isinstance(target_, str):
# Special case of a docstring; the function name is the first word.
target_name = target_.split('(', 1)[0]
else:
target_name = target_.__name__
with pyfile.open('w') as f:
f.write(helpers % (pypyroot, check_usemodules))
f.write('\n'.join(defs))
f.write('def %s():\n' % target_name)
f.write('\n'.join(source))
f.write("\n%s()\n" % target_name)
res, stdout, stderr = runsubprocess.run_subprocess(
python_, [str(pyfile)])
print pyfile.read()
print >> sys.stdout, stdout
print >> sys.stderr, stderr
if res == 81:
py.test.skip('%r was not compiled w/ required usemodules: %r' %
(python_, usemodules))
elif res > 0:
raise AssertionError("Subprocess failed:\n" + stderr)
def create_cffi_import_libraries(pypy_c,
options,
basedir,
only=None,
embed_dependencies=False):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(join(basedir, 'lib_pypy', '__pycache__')),
ignore_errors=True)
# be sure pip, setuptools are installed in a fresh pypy
# allows proper functioning of cffi on win32 with newer vc compilers
# XXX move this to a build slave step?
status, stdout, stderr = run_subprocess(str(pypy_c),
['-c', 'import setuptools'])
if status != 0:
status, stdout, stderr = run_subprocess(str(pypy_c),
['-m', 'ensurepip'])
failures = []
for key, module in cffi_build_scripts.items():
if only and key not in only:
print("* SKIPPING", key, '(not specified in --only)')
continue
if module is None or getattr(options, 'no_' + key, False):
continue
# the key is the module name, has it already been built?
status, stdout, stderr = run_subprocess(str(pypy_c),
['-c', 'import %s' % key])
if status == 0:
print('*', ' %s already built' % key, file=sys.stderr)
continue
if module.endswith('.py'):
args = [module]
cwd = str(join(basedir, 'lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
env = os.environ.copy()
print('*', ' '.join(args), file=sys.stderr)
if embed_dependencies:
curdir = os.path.abspath(os.path.dirname(__file__))
destdir = os.path.join(curdir, 'dest')
shutil.rmtree(destdir, ignore_errors=True)
os.makedirs(destdir)
if key == '_ssl' and sys.platform == 'darwin':
# this patch is loosely inspired by an Apple and adds
# a fallback to the OS X roots when none are available
patches = [
os.path.join(curdir,
'../../lib_pypy/_cffi_ssl/osx-roots.diff'),
]
else:
patches = []
status, stdout, stderr = _build_dependency(key,
destdir,
patches=patches)
if status != 0:
failures.append((key, module))
print("stdout:")
print(stdout.decode('utf-8'))
print("stderr:")
print(stderr.decode('utf-8'))
continue
env['CPPFLAGS'] = \
'-I{}/usr/include {}'.format(destdir, env.get('CPPFLAGS', ''))
env['LDFLAGS'] = \
'-L{}/usr/lib {}'.format(destdir, env.get('LDFLAGS', ''))
if key == '_ssl' and sys.platform == 'darwin':
# needed for our roots patch
env['LDFLAGS'] += ' -framework CoreFoundation -framework Security'
elif sys.platform == 'win32':
env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
try:
status, stdout, stderr = run_subprocess(str(pypy_c),
args,
cwd=cwd,
env=env)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
except:
import traceback
traceback.print_exc()
failures.append((key, module))
return failures
def _build_dependency(name, destdir, patches=[]):
import multiprocessing
import shutil
import subprocess
from rpython.tool.runsubprocess import run_subprocess
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
try:
url, dgst, args = cffi_dependencies[name]
except KeyError:
return 0, None, None
archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')
if not os.path.isdir(archive_dir):
os.makedirs(archive_dir)
archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])
# next, fetch the archive to disk, if needed
if not os.path.exists(archive) or _sha256(archive) != dgst:
print('fetching archive', url, file=sys.stderr)
urlretrieve(url, archive)
# extract the archive into our destination directory
print('unpacking archive', archive, file=sys.stderr)
_unpack_tarfile(archive, destdir)
sources = os.path.join(
destdir,
os.path.basename(archive)[:-7],
)
# apply any patches
if patches:
for patch in patches:
print('applying patch', patch, file=sys.stderr)
status, stdout, stderr = run_subprocess(
'/usr/bin/patch',
['-p1', '-i', patch],
cwd=sources,
)
if status != 0:
return status, stdout, stderr
print('configuring', sources, file=sys.stderr)
# configure & build it
status, stdout, stderr = run_subprocess(
'./configure',
[
'--prefix=/usr',
'--disable-shared',
'--enable-silent-rules',
'--disable-dependency-tracking',
] + args,
cwd=sources,
)
if status != 0:
return status, stdout, stderr
print('building', sources, file=sys.stderr)
status, stdout, stderr = run_subprocess(
'make',
[
'-s',
'-j' + str(multiprocessing.cpu_count()),
'install',
'DESTDIR={}/'.format(destdir),
],
cwd=sources,
)
return status, stdout, stderr
def create_cffi_import_libraries(pypy_c,
options,
basedir,
only=None,
embed_dependencies=False,
rebuild=False):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(join(basedir, 'lib_pypy', '__pycache__')),
ignore_errors=True)
# be sure pip, setuptools are installed in a fresh pypy
# allows proper functioning of cffi on win32 with newer vc compilers
# XXX move this to a build slave step?
status, stdout, stderr = run_subprocess(str(pypy_c),
['-c', 'import setuptools'])
if status != 0:
status, stdout, stderr = run_subprocess(str(pypy_c),
['-m', 'ensurepip'])
failures = []
for key, module in cffi_build_scripts.items():
if only and key not in only:
print("* SKIPPING", key, '(not specified in --only)')
continue
if module is None or getattr(options, 'no_' + key, False):
continue
if not rebuild:
# the key is the module name, has it already been built?
status, stdout, stderr = run_subprocess(
str(pypy_c), ['-c', 'import %s' % key])
if status == 0:
print('*', ' %s already built' % key, file=sys.stderr)
continue
if module.endswith('.py'):
args = [module]
cwd = str(join(basedir, 'lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
env = os.environ.copy()
print('*', ' '.join(args), file=sys.stderr)
if embed_dependencies:
status, stdout, stderr = _build_dependency(key)
if status != 0:
failures.append((key, module))
print("stdout:")
print(stdout.decode('utf-8'))
print("stderr:")
print(stderr.decode('utf-8'))
continue
env['CPPFLAGS'] = \
'-I{}/usr/include {}'.format(deps_destdir, env.get('CPPFLAGS', ''))
env['LDFLAGS'] = \
'-L{}/usr/lib {}'.format(deps_destdir, env.get('LDFLAGS', ''))
elif sys.platform == 'win32':
env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '')
env['LIB'] = r'..\externals\lib;' + env.get('LIB', '')
env['PATH'] = r'..\externals\bin;' + env.get('PATH', '')
try:
status, stdout, stderr = run_subprocess(str(pypy_c),
args,
cwd=cwd,
env=env)
if status != 0:
print(stdout, stderr, file=sys.stderr)
failures.append((key, module))
except:
import traceback
traceback.print_exc()
failures.append((key, module))
return failures
def create_cffi_import_libraries(pypy_c,
options,
basedir,
only=None,
embed_dependencies=False,
rebuild=False):
from rpython.tool.runsubprocess import run_subprocess
shutil.rmtree(str(join(basedir, 'lib_pypy', '__pycache__')),
ignore_errors=True)
env = os.environ
if sys.platform == 'win32':
# requires the repo pypy/externals to be located under pypy
externals_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'externals'))
assert os.path.exists(externals_path), externals_path
env = os.environ.copy()
env['INCLUDE'] = externals_path + r'\include;' + env.get('INCLUDE', '')
env['LIB'] = externals_path + r'\lib;' + env.get('LIB', '')
env['PATH'] = externals_path + r'\bin;' + env.get('PATH', '')
status, stdout, stderr = run_subprocess(str(pypy_c),
['-c', 'import setuptools'])
if status != 0:
status, stdout, stderr = run_subprocess(str(pypy_c),
['-m', 'ensurepip'])
failures = []
for key, module in cffi_build_scripts.items():
if only and key not in only:
print("* SKIPPING", key, '(not specified in --only)')
continue
if module is None or getattr(options, 'no_' + key, False):
continue
if not rebuild:
# the key is the module name, has it already been built?
status, stdout, stderr = run_subprocess(
str(pypy_c), ['-c', 'import %s' % key], env=env)
if status == 0:
print('*', ' %s already built' % key, file=sys.stderr)
continue
if module.endswith('.py'):
args = [module]
cwd = str(join(basedir, 'lib_pypy'))
else:
args = ['-c', 'import ' + module]
cwd = None
print('*', ' '.join(args), file=sys.stderr)
if embed_dependencies and key in cffi_dependencies:
status, stdout, stderr = _build_dependency(key)
if status != 0:
failures.append((key, module))
print("stdout:")
print(stdout.decode('utf-8'))
print("stderr:")
print(stderr.decode('utf-8'))
continue
env['CPPFLAGS'] = \
'-I{}/usr/include {}'.format(deps_destdir, env.get('CPPFLAGS', ''))
env['LDFLAGS'] = \
'-L{}/usr/lib {}'.format(deps_destdir, env.get('LDFLAGS', ''))
try:
status, stdout, stderr = run_subprocess(str(pypy_c),
args,
cwd=cwd,
env=env)
if status != 0:
print("stdout:")
print(stdout.decode('utf-8'), file=sys.stderr)
print("stderr:")
print(stderr.decode('utf-8'), file=sys.stderr)
raise RuntimeError('building {} failed'.format(key))
except:
import traceback
traceback.print_exc()
failures.append((key, module))
else:
# Make sure it worked
status, stdout, stderr = run_subprocess(
str(pypy_c),
['-c', "print('testing {0}'); import {0}".format(key)],
env=env)
if status != 0:
failures.append((key, module))
print("stdout:")
print(stdout.decode('utf-8'), file=sys.stderr)
print("stderr:")
print(stderr.decode('utf-8'), file=sys.stderr)
if os.path.exists(deps_destdir):
shutil.rmtree(deps_destdir, ignore_errors=True)
return failures
def run_with_python(python_, target_, usemodules, **definitions):
if python_ is None:
py.test.skip(
"Cannot find the default python3 interpreter to run with -A")
defs = []
for symbol, value in sorted(definitions.items()):
if isinstance(value, tuple) and isinstance(value[0], py.code.Source):
code, args = value
defs.append(str(code))
arg_repr = []
for arg in args:
if isinstance(arg, types.FunctionType):
arg_repr.append(arg.__name__)
elif isinstance(arg, types.MethodType):
arg_repr.append(arg.__name__)
else:
arg_repr.append(py3k_repr(arg))
args = ', '.join(arg_repr)
defs.append("self.%s = anonymous(%s)\n" % (symbol, args))
elif isinstance(value, types.MethodType):
# "def w_method(self)"
code = py.code.Code(value)
defs.append(str(code.source()))
defs.append("type(self).%s = %s\n" % (symbol, value.__name__))
elif isinstance(value, types.ModuleType):
name = value.__name__
defs.append("import %s; self.%s = %s\n" % (name, symbol, name))
elif isinstance(value, (str, unicode, int, long, float, list, tuple,
dict)) or value is None:
defs.append("self.%s = %s\n" % (symbol, py3k_repr(value)))
check_usemodules = ''
if usemodules:
usemodules = [_rename_module(name) for name in usemodules]
check_usemodules = """\
missing = set(%r).difference(sys.builtin_module_names)
if missing:
if not hasattr(sys, 'pypy_version_info'):
# They may be extension modules on CPython
name = None
for name in missing.copy():
if name in ['cpyext', '_cffi_backend', '_rawffi']:
missing.remove(name)
continue
try:
__import__(name)
except ImportError:
pass
else:
missing.remove(name)
del name
if missing:
sys.exit(81)
del missing""" % usemodules
source = list(py.code.Source(target_))
while source[0].startswith(('@py.test.mark.', '@pytest.mark.')):
source.pop(0)
source = source[1:]
pyfile = udir.join('src.py')
if isinstance(target_, str):
# Special case of a docstring; the function name is the first word.
target_name = target_.split('(', 1)[0]
else:
target_name = target_.__name__
with pyfile.open('w') as f:
f.write(helpers % (pypyroot, check_usemodules))
f.write('\n'.join(defs))
f.write('def %s():\n' % target_name)
f.write('\n'.join(source))
f.write("\ntry:\n %s()\n" % target_name)
f.write('finally:\n print("===aefwuiheawiu===")')
helper_dir = os.path.join(pypydir, 'tool', 'cpyext')
env = os.environ.copy()
env['PYTHONPATH'] = helper_dir
res, stdout, stderr = run_subprocess(python_, [str(pyfile)], env=env)
print pyfile.read()
print >> sys.stdout, stdout
print >> sys.stderr, stderr
if res == 81:
py.test.skip('%r was not compiled w/ required usemodules: %r' %
(python_, usemodules))
elif res != 0:
raise AssertionError("Subprocess failed with exit code %s:\n%s" %
(res, stderr))
elif "===aefwuiheawiu===" not in stdout:
raise AssertionError("%r crashed:\n%s" % (python_, stderr))
def run_with_python(python_, target_, usemodules, **definitions):
if python_ is None:
py.test.skip(
"Cannot find the default python3 interpreter to run with -A")
# we assume that the source of target_ is in utf-8. Unfortunately, we don't
# have any easy/standard way to determine from here the original encoding
# of the source file
helpers = r"""# -*- encoding: utf-8 -*-
if 1:
import sys
sys.path.append(%r)
%s
def skip(message):
print(message)
raise SystemExit(0)
__builtins__.skip = skip
__builtins__.py3k_skip = skip
class ExceptionWrapper:
pass
def raises(exc, func, *args, **kwargs):
import os
try:
if isinstance(func, str):
if func.startswith((' ', os.linesep)):
# it's probably an indented block, so we prefix if True:
# to avoid SyntaxError
func = "if True:\n" + func
frame = sys._getframe(1)
exec(func, frame.f_globals, frame.f_locals)
else:
func(*args, **kwargs)
except exc as e:
res = ExceptionWrapper()
res.value = e
return res
else:
raise AssertionError("DID NOT RAISE")
__builtins__.raises = raises
class Test:
pass
self = Test()
"""
defs = []
for symbol, value in sorted(definitions.items()):
if isinstance(value, tuple) and isinstance(value[0], py.code.Source):
code, args = value
defs.append(str(code))
arg_repr = []
for arg in args:
if isinstance(arg, types.FunctionType):
arg_repr.append(arg.__name__)
elif isinstance(arg, types.MethodType):
arg_repr.append(arg.__name__)
else:
arg_repr.append(py3k_repr(arg))
args = ', '.join(arg_repr)
defs.append("self.%s = anonymous(%s)\n" % (symbol, args))
elif isinstance(value, types.MethodType):
# "def w_method(self)"
code = py.code.Code(value)
defs.append(str(code.source()))
defs.append("type(self).%s = %s\n" % (symbol, value.__name__))
elif isinstance(value, types.ModuleType):
name = value.__name__
defs.append("import %s; self.%s = %s\n" % (name, symbol, name))
elif isinstance(value, (str, unicode, int, long, float, list, tuple,
dict)) or value is None:
defs.append("self.%s = %s\n" % (symbol, py3k_repr(value)))
check_usemodules = ''
if usemodules:
usemodules = [
str(RENAMED_USEMODULES.get(name, name)) for name in usemodules
]
check_usemodules = """\
missing = set(%r).difference(sys.builtin_module_names)
if missing:
if not hasattr(sys, 'pypy_version_info'):
# They may be extension modules on CPython
name = None
for name in missing.copy():
try:
__import__(name)
except ImportError:
pass
else:
missing.remove(name)
del name
if missing:
sys.exit(81)
del missing""" % usemodules
source = list(py.code.Source(target_))
while source[0].startswith(('@py.test.mark.', '@pytest.mark.')):
source.pop(0)
source = source[1:]
pyfile = udir.join('src.py')
if isinstance(target_, str):
# Special case of a docstring; the function name is the first word.
target_name = target_.split('(', 1)[0]
else:
target_name = target_.__name__
with pyfile.open('w') as f:
f.write(helpers % (pypyroot, check_usemodules))
f.write('\n'.join(defs))
f.write('def %s():\n' % target_name)
f.write('\n'.join(source))
f.write("\n%s()\n" % target_name)
res, stdout, stderr = runsubprocess.run_subprocess(python_, [str(pyfile)])
print pyfile.read()
print >> sys.stdout, stdout
print >> sys.stderr, stderr
if res == 81:
py.test.skip('%r was not compiled w/ required usemodules: %r' %
(python_, usemodules))
elif res > 0:
raise AssertionError("Subprocess failed:\n" + stderr)
def _build_dependency(name, patches=[]):
import shutil
from rpython.tool.runsubprocess import run_subprocess
try:
from urllib.request import urlretrieve
except ImportError:
from urllib import urlretrieve
try:
url, dgst, build_cmds = cffi_dependencies[name]
except KeyError:
return 0, None, None
archive_dir = os.path.join(tempfile.gettempdir(), 'pypy-archives')
if not os.path.isdir(archive_dir):
os.makedirs(archive_dir)
archive = os.path.join(archive_dir, url.rsplit('/', 1)[-1])
# next, fetch the archive to disk, if needed
if not os.path.exists(archive) or _sha256(archive) != dgst:
print('fetching archive', url, file=sys.stderr)
urlretrieve(url, archive)
# make sure the hash matches
if _sha256(archive) != dgst:
return 1, '{} archive {} hash mismatch'.format(name, archive), ''
shutil.rmtree(deps_destdir, ignore_errors=True)
os.makedirs(deps_destdir)
# extract the into our destination directory
print('unpacking archive', archive, file=sys.stderr)
_unpack_tarfile(archive, deps_destdir)
sources = os.path.join(
deps_destdir,
os.path.basename(archive).rsplit('.', 2)[0],
)
# apply any patches
if patches:
for patch in patches:
print('applying patch', patch, file=sys.stderr)
status, stdout, stderr = run_subprocess(
'/usr/bin/patch', ['-p1', '-i', patch], cwd=sources,
)
if status != 0:
return status, stdout, stderr
env = os.environ
if sys.platform == 'darwin':
target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
if target:
# override the value for building support libraries
env = os.environ.copy()
env['MACOSX_DEPLOYMENT_TARGET'] = target
print('setting MACOSX_DEPLOYMENT_TARGET to "{}"'.format(target))
for args in build_cmds:
print('running', ' '.join(args), 'in', sources, file=sys.stderr)
status, stdout, stderr = run_subprocess(args[0], args[1:],
cwd=sources, env=env)
if status != 0:
break
return status, stdout, stderr
