def test_setuptools_backend(self) -> None:
with volatile.dir() as d:
dp = Path(d)
Path(d, "pyproject.toml").write_text("")
requires, inst = get_backend(dp)
# self.assertEqual(["setuptools"], requires)
self.assertIsInstance(inst, SetuptoolsReader)
def manager(self):
"""Create a new postgres manager.
Any operation done to a postgres database must be performed inside
a manager. The manage will change the current `uid` for *all*
operations to `postgres`; for this reason it is important to not
perform any other operation while inside this contextmanager.
Furthermore, establishing a connection requires that OpenBSD-style
netcat is installed on the target system. Be aware that most
debian-systems install a "traditional" netcat by default (package
`netcat-traditional` instead of `netcat-openbsd`). Using traditional
netcat will result in errors proclaiming unexpected closing of the
socket.
"""
with ExitStack() as stack:
dtmp = stack.enter_context(volatile.dir())
sock_addr = os.path.join(dtmp, '.s.PGSQL.5432')
if self.ident:
stack.enter_context(proc.sudo(self.ident))
stack.enter_context(net.local_forward(self.remote_addr, sock_addr))
url = URL(drivername='postgresql',
username=self.user,
password=self.password,
database=self.database,
query={'host': dtmp})
engine = create_engine(url, echo=self.echo)
yield Manager(engine)
def sample_contents(s: AnyStr) -> Generator[str, None, None]:
with volatile.dir() as dtmp:
ptmp = Path(dtmp)
(ptmp / "pyproject.toml").write_text("")
if isinstance(s, bytes):
(ptmp / "sample.py").write_bytes(s)
else:
(ptmp / "sample.py").write_text(s)
yield dtmp
def test_str(self) -> None:
with volatile.dir() as d:
dp = Path(d).resolve()
# doesn't matter if it's a dir or file currently
(dp / "pyproject.toml").write_text("")
(dp / "x").mkdir()
self.assertEqual(dp, get_root(str(dp)))
self.assertEqual(dp, get_root(str(dp / "x")))
def test_various_indicators(self) -> None:
for ind in (".git", "pyproject.toml"):
with volatile.dir() as d:
dp = Path(d).resolve()
# doesn't matter if it's a dir or file currently
(dp / ind).write_text("")
self.assertEqual(dp, get_root(dp))
(dp / "x").mkdir()
self.assertEqual(dp, get_root(dp / "x"))
def test_dir_cleanup_after_exception():
try:
with volatile.dir() as dtmp:
name = dtmp
assert os.path.exists(name)
raise RuntimeError()
pass
except RuntimeError:
pass
assert not os.path.exists(name)
def test_flit_backend(self) -> None:
with volatile.dir() as d:
dp = Path(d)
Path(d, "pyproject.toml").write_text("""\
[build-system]
requires = ["flit_core >=2,<4"]
build-backend = "flit_core.buildapi"
""")
requires, inst = get_backend(dp)
self.assertEqual(["flit_core >=2,<4"], requires)
self.assertIsInstance(inst, FlitReader)
def _read(self, data: str, src_dir: str = ".") -> Distribution:
with volatile.dir() as d:
sp = Path(d, "setup.py")
sp.write_text(data)
Path(d, src_dir, "pkg").mkdir(parents=True)
Path(d, src_dir, "pkg", "__init__.py").touch()
Path(d, src_dir, "pkg", "sub").mkdir()
Path(d, src_dir, "pkg", "sub", "__init__.py").touch()
Path(d, src_dir, "pkg", "tests").mkdir()
Path(d, src_dir, "pkg", "tests", "__init__.py").touch()
return SetuptoolsReader(Path(d)).get_metadata()
def test_force_false_is_gentle():
try:
with pytest.raises(OSError):
with volatile.dir(force=False) as dtmp:
blocking_path = os.path.join(dtmp, 'blocking')
with open(blocking_path, 'w') as f:
f.write('hello')
finally:
assert os.path.exists(blocking_path)
os.unlink(blocking_path)
os.rmdir(os.path.dirname(blocking_path))
def test_optional_value(self) -> None:
with volatile.dir() as d:
dp = Path(d).resolve()
# doesn't matter if it's a dir or file currently
(dp / "pyproject.toml").write_text("")
(dp / "x").mkdir()
try:
prev = os.getcwd()
os.chdir(dp / "x")
self.assertEqual(dp, get_root())
finally:
os.chdir(prev)
def test_custom_root_indicators(self) -> None:
for iterable_type in [list, set, tuple]:
with volatile.dir() as d:
dp = Path(d).resolve()
# doesn't matter if it's a dir or file currently
(dp / ".git").write_text("")
self.assertEqual(
dp, get_root(dp, root_indicators=iterable_type([".git"])))
(dp / "x").mkdir()
self.assertEqual(
dp,
get_root(dp / "x",
root_indicators=iterable_type([".git"])))
def test_setup_py(self) -> None:
with volatile.dir() as d:
dp = Path(d)
(dp / "setup.py").write_text(
"""\
from setuptools import setup
the_name = "foo"
setup(name=the_name, install_requires=["abc"], setup_requires=["def"])
"""
)
r = SetuptoolsReader(dp)
self.assertEqual(("setuptools", "def"), r.get_requires_for_build_sdist())
self.assertEqual(
("setuptools", "wheel", "def"), r.get_requires_for_build_wheel()
)
def generate_self_signed_cert(hostname):
# FIXME: instead of openssl, use cryptography/nacl?
with volatile.dir() as tmp:
log.info('Generating new temporary self-signed certificate')
subprocess.check_output([
'openssl', 'req', '-x509', '-subj', '/CN=' + hostname, '-nodes',
'-newkey', 'rsa:4096', '-keyout', 'snakeoil.pem', '-out',
'snakeoil.crt', '-days', '7'
],
cwd=tmp,
stderr=subprocess.STDOUT)
cert = open(os.path.join(tmp, 'snakeoil.crt')).read()
key = open(os.path.join(tmp, 'snakeoil.pem')).read()
return key, cert
def test_fail(self) -> None:
with volatile.dir() as d:
pd = Path(d)
(pd / "fake_module").mkdir()
(pd / "fake_module" / "__init__.py").write_text("")
(pd / "fake_module" / "x.py").write_text("x x")
output, err = self.do_run([f"--root={d}", "fake_module"])
self.assertEqual(
"""\
fake_module ok
""",
output,
)
# TODO message
self.assertIsInstance(err, SyntaxError)
def test_all(self) -> None:
with volatile.dir() as d:
Path(d, "setup.py").write_text("""\
from setuptools import setup
setup(
setup_requires=[
"a; python_version < '3'", "b"],
install_requires=[
"c; python_version < '3'", "d"],
)
""")
env = EnvironmentMarkers.for_python("3.6.0")
self.assertEqual(["setuptools", "b"],
get_requires_for_build_sdist(Path(d), env))
self.assertEqual(["setuptools", "wheel", "b"],
get_requires_for_build_wheel(Path(d), env))
def manager(self):
with ExitStack() as stack:
dtmp = stack.enter_context(volatile.dir())
sock_addr = os.path.join(dtmp, '.s.PGSQL.5432')
if self.ident:
stack.enter_context(proc.sudo(self.ident))
stack.enter_context(net.local_forward(self.remote_addr, sock_addr))
url = URL(drivername='postgresql',
username=self.user,
password=self.password,
database=self.database,
query={'host': dtmp})
engine = create_engine(url, echo=self.echo)
yield Manager(engine)
def test_setup_cfg_dash_normalization(self) -> None:
# I can't find documentation for this, but e.g. auditwheel 3.2.0 uses
# dashes instead of underscores and it works.
with volatile.dir() as d:
dp = Path(d)
(dp / "setup.cfg").write_text(
"""\
[metadata]
name = foo
author = Foo
author-email = [email protected]
"""
)
r = SetuptoolsReader(dp)
md = r.get_metadata()
self.assertEqual("*****@*****.**", md.author_email)
def get_metadata(path: Path) -> Message:
with volatile.dir() as d:
build_sys = compat_system(path)
hooks = Pep517HookCaller(
path,
build_backend=build_sys["build-backend"],
backend_path=build_sys.get("backend-path"),
)
if build_sys.get("requires"):
subprocess.run([sys.executable, "-m", "pip", "install"] +
build_sys["requires"])
dist_info = hooks.prepare_metadata_for_build_wheel(d)
metadata_path = Path(d, dist_info, "METADATA")
with open(metadata_path) as fp:
return Parser().parse(fp)
def test_setup_cfg(self) -> None:
with volatile.dir() as d:
dp = Path(d)
(dp / "setup.cfg").write_text(
"""\
[metadata]
name = foo
[options]
install_requires = abc
setup_requires = def
"""
)
r = SetuptoolsReader(dp)
self.assertEqual(("setuptools", "def"), r.get_requires_for_build_sdist())
self.assertEqual(
("setuptools", "wheel", "def"), r.get_requires_for_build_wheel()
)
def test_basic(self) -> None:
with volatile.dir() as d:
dp = Path(d)
(dp / "pyproject.toml").write_text("""\
[build-system]
requires = ["poetry-core>=1.0.0a9"]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "Name"
version = "1.5.2"
description = "Short Desc"
authors = ["Author <*****@*****.**>"]
license = "BSD-3-Clause"
homepage = "http://example.com"
classifiers = [
"Not a real classifier",
]
[tool.poetry.dependencies]
python = "~2.7 || ^3.5"
functools32 = { version = "^3.2.3", python = "~2.7" }
[tool.poetry.urls]
"Bug Tracker" = "https://github.com/python-poetry/poetry/issues"
""")
r = PoetryReader(dp)
md = r.get_metadata()
self.assertEqual("Name", md.name)
self.assertEqual("1.5.2", md.version)
self.assertEqual("BSD-3-Clause", md.license)
self.assertEqual(
{
"homepage":
"http://example.com",
"Bug Tracker":
"https://github.com/python-poetry/poetry/issues",
},
md.project_urls,
)
self.assertEqual(["Not a real classifier"], md.classifiers)
self.assertEqual(["functools32"], md.requires_dist)
def test_simplest(self) -> None:
with volatile.dir() as d:
dp = Path(d)
(dp / "pyproject.toml").write_text(
"""\
[build-system]
build-backend = "flit_core.buildapi"
[tool.flit.metadata]
name = "Name"
"""
)
# I assume this would be an error in flit, but we want to make sure we
# handle missing metadata appropriately.
r = FlitReader(dp)
self.assertEqual((), r.get_requires_for_build_sdist())
self.assertEqual((), r.get_requires_for_build_wheel())
md = r.get_metadata()
self.assertEqual("Name", md.name)
def test_multiple_modules_ok(self) -> None:
with volatile.dir() as d:
pd = Path(d)
(pd / "fake_module_1").mkdir()
(pd / "fake_module_1" / "__init__.py").write_text("")
(pd / "fake_module_2").mkdir()
(pd / "fake_module_2" / "__init__.py").write_text("")
output, err = self.do_run(
[f"--root={d}", "fake_module_1", "fake_module_2"])
self.assertEqual(
"""\
fake_module_1 ok
fake_module_2 ok
""",
output,
)
self.assertEqual(None, err)
def test_normal(self) -> None:
with volatile.dir() as d:
dp = Path(d)
(dp / "pyproject.toml").write_text(
"""\
[build-system]
requires = ["flit_core >=2,<4"]
build-backend = "flit_core.buildapi"
[tool.flit.metadata]
name = "Name"
module = "foo"
requires = ["abc", "def"]
[tool.flit.metadata.urls]
Foo = "https://"
"""
)
(dp / "foo").mkdir()
(dp / "foo" / "tests").mkdir()
(dp / "foo" / "__init__.py").write_text("")
(dp / "foo" / "tests" / "__init__.py").write_text("")
r = FlitReader(dp)
# Notably these do not include flit itself; that's handled by
# dowsing.pep517
self.assertEqual(["abc", "def"], r.get_requires_for_build_sdist())
self.assertEqual(["abc", "def"], r.get_requires_for_build_wheel())
md = r.get_metadata()
self.assertEqual("Name", md.name)
self.assertEqual(
{
"metadata_version": "2.1",
"name": "Name",
"packages": ["foo", "foo.tests"],
"packages_dict": {"foo": "foo", "foo.tests": "foo/tests"},
"requires_dist": ["abc", "def"],
"project_urls": {"Foo": "https://"},
},
md.asdict(),
)
def test_single_modules_ok(self) -> None:
with volatile.dir() as d:
pd = Path(d)
(pd / "fake_module").mkdir()
(pd / "fake_module" / "__init__.py").write_text("")
(pd / "fake_module" / "__main__.py").write_text("")
(pd / "fake_module" / "tests").mkdir()
(pd / "fake_module" / "tests" / "__init__.py").write_text("")
(pd / "fake_module" / "tests" / "x.py").write_text("")
(pd / "fake_module" / "x.py").write_text("")
output, err = self.do_run([f"--root={d}", "fake_module"])
self.assertEqual(
"""\
fake_module ok
fake_module.x ok
""",
output,
)
self.assertEqual(None, err)
def test_invalid_dir_raises(self) -> None:
with volatile.dir() as d:
dp = Path(d).resolve()
with self.assertRaisesRegex(ValueError,
"missing is not a directory"):
get_root(dp / "missing")
(dp / "file").write_text("")
with self.assertRaisesRegex(ValueError, "file is not a directory"):
get_root(dp / "file")
# This can be different depending on whether a tmpfs is used
with self.assertRaisesRegex(
ValueError,
"No root found (before actual root|on same device)"):
get_root(dp)
with self.assertRaisesRegex(
ValueError,
"No root found (before actual root|on same device)"):
get_root(Path("/"))
def test_orjson(self) -> None:
# This is a simplified version of orjson 3.4.0
with volatile.dir() as d:
dp = Path(d)
(dp / "pyproject.toml").write_text("""\
[project]
name = "orjson"
repository = "https://example.com/"
[build-system]
build-backend = "maturin"
requires = ["maturin>=0.8.1,<0.9"]
""")
(dp / "Cargo.toml").write_text("""\
[package]
name = "orjson"
version = "3.4.0"
authors = ["foo <*****@*****.**>"]
description = "Summary here"
license = "Apache-2.0 OR MIT"
repository = "https://example.com/repo"
homepage = "https://example.com/home"
readme = "README.md"
keywords = ["foo", "bar", "baz"]
[package.metadata.maturin]
requires-python = ">=3.6"
classifer = [
"License :: OSI Approved :: Apache Software License",
"License :: OSI Approved :: MIT License",
]
""")
r = MaturinReader(dp)
md = r.get_metadata()
self.assertEqual("orjson", md.name)
self.assertEqual("3.4.0", md.version)
def repo_dir():
with volatile.dir() as dtmp:
subprocess.check_call(['git', 'init', dtmp])
yield dtmp
def local_forward(remote_addr, local_addr=('127.0.0.1', 0)):
class Handler(SocketServer.BaseRequestHandler):
@keep_context
def handle(self):
log.debug('New connection to {} from {}'.format(
remote_addr, self.request.getpeername()))
if isinstance(remote_addr, tuple):
# remote is a tcp address
chan = my_remote.tcp_connect(remote_addr)
else:
# remote is a unix socket
chan = my_remote.unix_connect(remote_addr)
while True:
r, _, _ = select.select([self.request, chan], [], [])
if self.request in r:
buf = self.request.recv(4096)
if len(buf) == 0:
# client closed connection, we're done
break
chan.send(buf)
if chan in r:
buf = chan.recv(4096)
if len(buf) == 0:
# server closed connection
break
self.request.send(buf)
log.debug('Closed connection to {} from {}'.format(
remote_addr, self.request.getpeername()))
with contextlib2.ExitStack() as stack:
if isinstance(local_addr, tuple):
server = SocketServer.ThreadingTCPServer(local_addr, Handler)
else:
# assume its a string, denoting a unix domain socket
if not local_addr:
dtmp = stack.enter_context(volatile.dir())
local_addr = os.path.join(dtmp, 'remote.sock')
server = SocketServer.ThreadingUnixStreamServer(
local_addr, Handler)
server.allow_reuse_address = False
server.daemon_threads = True
my_remote = remote._get_current_object()
t = threading.Thread(target=keep_context(server.serve_forever))
t.daemon = True
t.start()
# wait for the server to shutdown
stack.callback(t.join)
stack.callback(server.shutdown)
stack.callback(
log.debug,
'Waiting for shutdown of {}'.format(server.server_address))
log.debug('Forward established: {} => {}'.format(
server.server_address, remote_addr))
yield server.server_address
def test_main(self):
with volatile.dir():
f = Foo()
def test_temp_dir():
with volatile.dir() as dtmp:
assert os.path.exists(dtmp)
assert os.path.isdir(dtmp)
assert not os.path.exists(dtmp)
def test_can_remove_dir_without_error():
with volatile.dir() as dtmp:
os.rmdir(dtmp)
def test_no_backend(self) -> None:
with volatile.dir() as d:
dp = Path(d)
requires, inst = get_backend(dp)
# self.assertEqual(["setuptools"], requires)
self.assertIsInstance(inst, SetuptoolsReader)
def local_forward(remote_addr, local_addr=('127.0.0.1', 0)):
class Handler(SocketServer.BaseRequestHandler):
@keep_context
def handle(self):
log.debug('New connection to {} from {}'.format(
remote_addr, self.request.getpeername()))
if isinstance(remote_addr, tuple):
# remote is a tcp address
chan = my_remote.tcp_connect(remote_addr)
else:
# remote is a unix socket
chan = my_remote.unix_connect(remote_addr)
while True:
r, _, _ = select.select([self.request, chan], [], [])
if self.request in r:
buf = self.request.recv(4096)
if len(buf) == 0:
# client closed connection, we're done
break
chan.send(buf)
if chan in r:
buf = chan.recv(4096)
if len(buf) == 0:
# server closed connection
break
self.request.send(buf)
log.debug('Closed connection to {} from {}'.format(
remote_addr, self.request.getpeername()))
with contextlib2.ExitStack() as stack:
if isinstance(local_addr, tuple):
server = SocketServer.ThreadingTCPServer(local_addr, Handler)
else:
# assume its a string, denoting a unix domain socket
if not local_addr:
dtmp = stack.enter_context(volatile.dir())
local_addr = os.path.join(dtmp, 'remote.sock')
server = SocketServer.ThreadingUnixStreamServer(local_addr,
Handler)
server.allow_reuse_address = False
server.daemon_threads = True
my_remote = remote._get_current_object()
t = threading.Thread(target=keep_context(server.serve_forever))
t.daemon = True
t.start()
# wait for the server to shutdown
stack.callback(t.join)
stack.callback(server.shutdown)
stack.callback(
log.debug,
'Waiting for shutdown of {}'.format(server.server_address))
log.debug('Forward established: {} => {}'.format(
server.server_address, remote_addr))
yield server.server_address
