def uwsgi_context(*args, **kwargs):
"""A context manager that provides a temporary directory with a pex-bootstrapped uwsgi.
Usage:
import os
from pyuwsgi import UWSGI_BINARY_PATH, uwsgi_context
with uwsgi_context() as uwsgi_path:
uwsgi_bin_path = os.path.join(uwsgi_path, UWSGI_BINARY_PATH)
p = subprocess.Popen([uwsgi_bin_path, '--arg1', '--arg2'])
...
>>> with uwsgi_context() as uwsgi_path:
... print uwsgi_path
... os.listdir(uwsgi_path)
...
/private/var/folders/3t/xkwqrkld4xxgklk2s4n41jb80000gn/T/tmpCY4JhN
['pex_uwsgi']
"""
old_cwd = os.getcwd()
with temporary_dir(*args, **kwargs) as temp_dir:
with pushd(temp_dir):
write_resource(UWSGI_BINARY_PATH)
write_resource(UWSGI_BOOTSTRAP_PATH)
with pushd(old_cwd):
yield temp_dir
def setUpClass(cls):
cls.origin = safe_mkdtemp()
with pushd(cls.origin):
subprocess.check_call(['git', 'init', '--bare'])
cls.gitdir = safe_mkdtemp()
cls.worktree = safe_mkdtemp()
cls.readme_file = os.path.join(cls.worktree, 'README')
with environment_as(GIT_DIR=cls.gitdir, GIT_WORK_TREE=cls.worktree):
cls.init_repo('depot', cls.origin)
touch(cls.readme_file)
subprocess.check_call(['git', 'add', 'README'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master'])
with safe_open(cls.readme_file, 'w') as readme:
readme.write('Hello World.')
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
cls.clone2 = safe_mkdtemp()
with pushd(cls.clone2):
cls.init_repo('origin', cls.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
cls.git = Git(gitdir=cls.gitdir, worktree=cls.worktree)
def uwsgi_context(*args, **kwargs):
"""A context manager that provides a temporary directory with a pex-bootstrapped uwsgi.
Usage:
import os
from pyuwsgi import UWSGI_BINARY_PATH, uwsgi_context
with uwsgi_context() as uwsgi_path:
uwsgi_bin_path = os.path.join(uwsgi_path, UWSGI_BINARY_PATH)
p = subprocess.Popen([uwsgi_bin_path, '--arg1', '--arg2'])
...
>>> with uwsgi_context() as uwsgi_path:
... print uwsgi_path
... os.listdir(uwsgi_path)
...
/private/var/folders/3t/xkwqrkld4xxgklk2s4n41jb80000gn/T/tmpCY4JhN
['pex_uwsgi']
"""
old_cwd = os.getcwd()
with temporary_dir(*args, **kwargs) as temp_dir:
with pushd(temp_dir):
write_resource(UWSGI_BINARY_PATH)
write_resource(UWSGI_BOOTSTRAP_PATH)
with pushd(old_cwd):
yield temp_dir
def test_unwriteable_contents():
my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=['my_app'],
include_package_data=True,
package_data={'my_app': ['unwriteable.so']},
)
""")
UNWRITEABLE_PERMS = 0o400
with temporary_content(
{
'setup.py': my_app_setup_py,
'my_app/__init__.py': '',
'my_app/unwriteable.so': ''
},
perms=UNWRITEABLE_PERMS) as my_app_project_dir:
with pushd(my_app_project_dir):
subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel'])
uses_my_app_setup_py = dedent("""
from setuptools import setup
setup(
name='uses_my_app',
version='0.0.0',
zip_safe=True,
install_requires=['my_app'],
)
""")
with temporary_content({'setup.py': uses_my_app_setup_py
}) as uses_my_app_project_dir:
with pushd(uses_my_app_project_dir):
subprocess.check_call([
sys.executable, 'setup.py', 'bdist_pex',
'--pex-args=--disable-cache --no-pypi -f {}'.format(
os.path.join(my_app_project_dir, 'dist'))
])
with open_zip('dist/uses_my_app-0.0.0.pex') as zf:
unwriteable_sos = [
path for path in zf.namelist()
if path.endswith('my_app/unwriteable.so')
]
assert 1 == len(unwriteable_sos)
unwriteable_so = unwriteable_sos.pop()
zf.extract(unwriteable_so)
assert UNWRITEABLE_PERMS == stat.S_IMODE(
os.stat(unwriteable_so).st_mode)
def test_nested_pushd():
pre_cwd = os.getcwd()
with temporary_dir() as tempdir1:
with pushd(tempdir1) as path1:
assert os.getcwd() == os.path.realpath(tempdir1)
with temporary_dir(root_dir=tempdir1) as tempdir2:
with pushd(tempdir2) as path2:
assert os.getcwd() == os.path.realpath(tempdir2)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == pre_cwd
assert os.getcwd() == pre_cwd
def test_via_pantsini(self):
with temporary_dir() as root:
root = os.path.realpath(root)
touch(os.path.join(root, 'pants.ini'))
with pushd(root):
self.assertEqual(root, BuildRoot().path)
BuildRoot().reset()
child = os.path.join(root, 'one', 'two')
safe_mkdir(child)
with pushd(child):
self.assertEqual(root, BuildRoot().path)
def test_via_pantsini(self):
with temporary_dir() as root:
root = os.path.realpath(root)
touch(os.path.join(root, 'pants.ini'))
with pushd(root):
self.assertEqual(root, BuildRoot().path)
BuildRoot().reset()
child = os.path.join(root, 'one', 'two')
safe_mkdir(child)
with pushd(child):
self.assertEqual(root, BuildRoot().path)
def test_nested_pushd():
pre_cwd = os.getcwd()
with temporary_dir() as tempdir1:
with pushd(tempdir1) as path1:
assert os.getcwd() == os.path.realpath(tempdir1)
with temporary_dir(root_dir=tempdir1) as tempdir2:
with pushd(tempdir2) as path2:
assert os.getcwd() == os.path.realpath(tempdir2)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == pre_cwd
assert os.getcwd() == pre_cwd
def build_egg(self, egg_root, target):
"""Build an egg containing the files at egg_root for the specified target.
There must be an egg_root/setup.py file."""
# TODO(Brian Wickman): Do a sanity check somewhere to ensure that
# setuptools is on the path?
args = [
sys.executable, 'setup.py', 'bdist_egg', '--dist-dir=dist',
'--bdist-dir=build.%s' % target.name
]
with pushd(egg_root):
with environment_as(PYTHONPATH=':'.join(sys.path)):
po = subprocess.Popen(args,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
rv = po.wait()
eggs = os.path.abspath(os.path.join('dist', '*.egg'))
eggs = glob.glob(eggs)
if rv != 0 or len(eggs) != 1:
comm = po.communicate()
print('egg generation failed (return value=%d, num eggs=%d)' %
(rv, len(eggs)),
file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
raise EggBuilder.EggBuildingException(
'Generation of eggs failed for target = %s' % target)
egg_path = eggs[0]
return egg_path
def assert_entry_points(entry_points):
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
entry_points=%(entry_points)r,
)
""" % dict(entry_points=entry_points))
my_app = dedent("""
def do_something():
print("hello world!")
""")
with temporary_content({
'setup.py': setup_py,
'my_app.py': my_app
}) as project_dir:
with pushd(project_dir):
subprocess.check_call([sys.executable, 'setup.py', 'bdist_pex'])
process = subprocess.Popen(
[os.path.join(project_dir, 'dist', 'my_app-0.0.0.pex')],
stdout=subprocess.PIPE)
stdout, _ = process.communicate()
assert '{pex_root}' not in os.listdir(project_dir)
assert 0 == process.returncode
assert stdout == b'hello world!\n'
def assert_entry_points(entry_points):
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
entry_points=%(entry_points)r,
)
""" % dict(entry_points=entry_points))
my_app = dedent("""
def do_something():
print("hello world!")
""")
with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir:
with pushd(project_dir):
subprocess.check_call([sys.executable, 'setup.py', 'bdist_pex'])
process = subprocess.Popen([os.path.join(project_dir, 'dist', 'my_app-0.0.0.pex')],
stdout=subprocess.PIPE)
stdout, _ = process.communicate()
assert 0 == process.returncode
assert stdout == b'hello world!\n'
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def build_egg(self, egg_root, target):
"""Build an egg containing the files at egg_root for the specified target.
There must be an egg_root/setup.py file."""
# TODO(Brian Wickman): Do a sanity check somewhere to ensure that
# setuptools is on the path?
args = [
sys.executable,
'setup.py', 'bdist_egg',
'--dist-dir=dist',
'--bdist-dir=build.%s' % target.name]
with pushd(egg_root):
print 'EggBuilder executing: %s' % ' '.join(args)
with environment_as(PYTHONPATH = ':'.join(sys.path)):
po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
rv = po.wait()
eggs = os.path.abspath(os.path.join('dist', '*.egg'))
eggs = glob.glob(eggs)
if rv != 0 or len(eggs) != 1:
comm = po.communicate()
print >> sys.stderr, 'egg generation failed (return value=%d, num eggs=%d)' % (
rv, len(eggs))
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
raise EggBuilder.EggBuildingException(
'Generation of eggs failed for target = %s' % target)
egg_path = eggs[0]
return egg_path
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def test(self):
self.assertEqual(set(), self.git.changed_files())
self.assertEqual(set(['README']),
self.git.changed_files(from_commit='HEAD^'))
tip_sha = self.git.commit_id
self.assertTrue(tip_sha)
self.assertTrue(tip_sha in self.git.changelog())
self.assertTrue(self.git.tag_name.startswith('first-'))
self.assertEqual('master', self.git.branch_name)
def edit_readme():
with open(self.readme_file, 'a') as readme:
readme.write('More data.')
edit_readme()
with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
untracked.write('make install')
self.assertEqual(set(['README']), self.git.changed_files())
self.assertEqual(set(['README', 'INSTALL']),
self.git.changed_files(include_untracked=True))
try:
# These changes should be rejected because our branch point from origin is 1 commit behind
# the changes pushed there in clone 2.
self.git.commit('API Changes.')
except Scm.RemoteException:
with environment_as(GIT_DIR=self.gitdir,
GIT_WORK_TREE=self.worktree):
subprocess.check_call(
['git', 'reset', '--hard', 'depot/master'])
self.git.refresh()
edit_readme()
self.git.commit('''API '"' " Changes.''')
self.git.tag('second', message='''Tagged ' " Changes''')
with temporary_dir() as clone:
with pushd(clone):
self.init_repo('origin', self.origin)
subprocess.check_call(
['git', 'pull', '--tags', 'origin', 'master:master'])
with open(os.path.realpath('README')) as readme:
self.assertEqual('--More data.', readme.read())
git = Git()
# Check that we can pick up committed and uncommitted changes.
with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
changes.write('none')
subprocess.check_call(['git', 'add', 'CHANGES'])
self.assertEqual(set(['README', 'CHANGES']),
git.changed_files(from_commit='first'))
self.assertEqual('master', git.branch_name)
self.assertEqual('second', git.tag_name)
def test_simple_pushd():
pre_cwd = os.getcwd()
with temporary_dir() as tempdir:
with pushd(tempdir) as path:
assert path == tempdir
assert os.getcwd() == os.path.realpath(tempdir)
assert os.getcwd() == pre_cwd
assert os.getcwd() == pre_cwd
def test_simple_pushd():
pre_cwd = os.getcwd()
with temporary_dir() as tempdir:
with pushd(tempdir) as path:
assert path == tempdir
assert os.getcwd() == os.path.realpath(tempdir)
assert os.getcwd() == pre_cwd
assert os.getcwd() == pre_cwd
def test_parse_from_sub_dir(self):
with self.workspace('a/b/c/BUILD') as root_dir:
with pushd(os.path.join(root_dir, 'a')):
self.assertAddress(root_dir, 'a/b/c/BUILD', 'c',
Address.parse(root_dir, 'b/c', is_relative=True))
with pytest.raises(IOError):
Address.parse(root_dir, 'b/c', is_relative=False)
def test_parse_from_sub_dir(self):
with self.workspace('a/b/c/BUILD') as root_dir:
with pushd(os.path.join(root_dir, 'a')):
self.assertAddress(
root_dir, 'a/b/c/BUILD', 'c',
Address.parse(root_dir, 'b/c', is_relative=True))
with pytest.raises(IOError):
Address.parse(root_dir, 'b/c', is_relative=False)
def test_upload_with_docs(self):
users = {"user": {"password": "******"}}
indices = {"user/index": {}}
with TestServer(users, indices) as devpi:
devpi.login("user", "secret")
devpi.use("user/index")
with pushd('tests/fixture/package'):
devpi.upload(with_docs=True)
def test_upload_with_docs(self):
users = {"user": {"password": "******"}}
indices = {"user/index": {}}
with TestServer(users, indices) as devpi:
devpi.login("user", "secret")
devpi.use("user/index")
with pushd('tests/fixture/package'):
devpi.upload(with_docs=True)
def test(self):
self.assertEqual(set(), self.git.changed_files())
self.assertEqual(set(['README']), self.git.changed_files(from_commit='HEAD^'))
tip_sha = self.git.commit_id
self.assertTrue(tip_sha)
self.assertTrue(tip_sha in self.git.changelog())
self.assertTrue(self.git.tag_name.startswith('first-'))
self.assertEqual('master', self.git.branch_name)
def edit_readme():
with open(self.readme_file, 'a') as readme:
readme.write('More data.')
edit_readme()
with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
untracked.write('make install')
self.assertEqual(set(['README']), self.git.changed_files())
self.assertEqual(set(['README', 'INSTALL']), self.git.changed_files(include_untracked=True))
try:
# These changes should be rejected because our branch point from origin is 1 commit behind
# the changes pushed there in clone 2.
self.git.commit('API Changes.')
except Scm.RemoteException:
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
subprocess.check_call(['git', 'reset', '--hard', 'depot/master'])
self.git.refresh()
edit_readme()
self.git.commit('''API '"' " Changes.''')
self.git.tag('second', message='''Tagged ' " Changes''')
with temporary_dir() as clone:
with pushd(clone):
subprocess.check_call(['git', 'init'])
subprocess.check_call(['git', 'remote', 'add', 'origin', self.origin])
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with open(os.path.realpath('README')) as readme:
self.assertEqual('--More data.', readme.read())
git = Git()
# Check that we can pick up committed and uncommitted changes.
with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
changes.write('none')
subprocess.check_call(['git', 'add', 'CHANGES'])
self.assertEqual(set(['README', 'CHANGES']), git.changed_files(from_commit='first'))
self.assertEqual('master', git.branch_name)
self.assertEqual('second', git.tag_name)
def dump(self, jarpath, jarfile):
self.context.log.debug(' dumping %s' % jarpath)
with temporary_dir() as tmpdir:
with pushd(tmpdir):
with self.open_jar(jarpath) as sourcejar:
sourcejar.extractall()
for root, dirs, files in os.walk(tmpdir):
for file in files:
path = os.path.join(root, file)
relpath = os.path.relpath(path, tmpdir)
if Manifest.PATH != relpath:
jarfile.write(path, relpath)
def _resolve_paths(self, rel_base, paths):
"""
Resolves paths relative to the given rel_base from the build root.
For example:
target: ~/workspace/src/java/com/twitter/common/base/BUILD
rel_base: src/resources
Resolves paths from:
~/workspace/src/resources/com/twitter/common/base
"""
# meta targets are composed of already-resolved paths
if not paths or self.is_meta:
return paths
def flatten_paths(*items):
"""Flattens one or more items into a list. If the item is iterable each of its items is
flattened. If an item is callable, it is called and the result is flattened. Otherwise the
atom is appended to the flattened list. These rules are applied recursively such that the
returned list will only contain non-iterable, non-callable atoms."""
flat = []
def flatmap(item):
if isinstance(item, Compatibility.string):
flat.append(item)
else:
try:
for i in iter(item):
flatmap(i)
except:
if callable(item):
flatmap(item())
else:
flat.append(item)
for item in items:
flatmap(item)
return flat
src_relpath = os.path.relpath(
self.address.buildfile.parent_path,
os.path.join(get_buildroot(), self.target_base))
resolve_basepath = os.path.join(get_buildroot(), rel_base, src_relpath)
with pushd(resolve_basepath):
return [
os.path.normpath(os.path.join(src_relpath, path))
for path in flatten_paths(paths)
]
def repo(tmpdir):
with pushd(tmpdir.strpath):
repo = git.Repo.init(tmpdir.strpath)
filename = 'test'
tmpdir.join(filename).write(first_commit_file_content)
repo.index.add([filename])
repo.index.commit('initial commit')
repo.create_head('a')
tmpdir.join(filename).write('more content')
repo.index.add([filename])
repo.index.commit('second commit')
return repo
def repo(tmpdir):
with pushd(tmpdir.strpath):
repo = git.Repo.init(tmpdir.strpath)
filename = 'test'
tmpdir.join(filename).write(first_commit_file_content)
repo.index.add([filename])
repo.index.commit('initial commit')
repo.create_head('a')
tmpdir.join(filename).write('more content')
repo.index.add([filename])
repo.index.commit('second commit')
return repo
def _resolve_paths(self, rel_base, paths):
"""Resolves paths relative to the given rel_base from the build root.
For example:
target: ~/workspace/src/java/com/twitter/common/base/BUILD
rel_base: src/resources
Resolves paths from:
~/workspace/src/resources/com/twitter/common/base
"""
if not paths:
return []
def flatten_paths(*items):
"""Flattens one or more items into a list.
If the item is iterable each of its items is flattened. If an item is callable, it is called
and the result is flattened. Otherwise the atom is appended to the flattened list. These
rules are applied recursively such that the returned list will only contain non-iterable,
non-callable atoms.
"""
flat = []
def flatmap(item):
if isinstance(item, Compatibility.string):
flat.append(item)
else:
try:
for i in iter(item):
flatmap(i)
except TypeError:
if callable(item):
flatmap(item())
else:
flat.append(item)
for item in items:
flatmap(item)
return flat
src_relpath = os.path.relpath(
self.address.buildfile.parent_path, os.path.join(get_buildroot(), self.target_base)
)
resolve_basepath = os.path.join(get_buildroot(), rel_base, src_relpath)
with pushd(resolve_basepath):
return [os.path.normpath(os.path.join(src_relpath, path)) for path in flatten_paths(paths)]
def execute(self):
config = Config.load()
distdir = config.getdefault('pants_distdir')
setup_dir = os.path.join(
distdir, '%s-%s' %
(self.target.provides._name, self.target.provides._version))
chroot = Chroot(distdir, name=self.target.provides._name)
self.write_sources(chroot)
self.write_setup(chroot)
if os.path.exists(setup_dir):
import shutil
shutil.rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run
or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
po.wait()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in po.stdout.read().splitlines():
print('stdout: %s' % line)
for line in po.stderr.read().splitlines():
print('stderr: %s' % line)
return po.returncode
expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name,
self.target.provides._version)
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(distdir, expected_tgz)
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
print('Wrote %s' % dist_tgz)
safe_rmtree(setup_dir)
def test_restful_cache(self):
httpd = None
httpd_thread = None
try:
with temporary_dir() as cache_root:
with pushd(cache_root): # SimpleRESTHandler serves from the cwd.
httpd = SocketServer.TCPServer(("localhost", 0), SimpleRESTHandler)
port = httpd.server_address[1]
httpd_thread = Thread(target=httpd.serve_forever)
httpd_thread.start()
with temporary_dir() as artifact_root:
artifact_cache = RESTfulArtifactCache(MockLogger(), artifact_root, "http://localhost:%d" % port)
self.do_test_artifact_cache(artifact_cache)
finally:
if httpd:
httpd.shutdown()
if httpd_thread:
httpd_thread.join()
def test_restful_cache(self):
httpd = None
httpd_thread = None
try:
with temporary_dir() as cache_root:
with pushd(cache_root): # SimpleRESTHandler serves from the cwd.
httpd = SocketServer.TCPServer(('localhost', 0), SimpleRESTHandler)
port = httpd.server_address[1]
httpd_thread = Thread(target=httpd.serve_forever)
httpd_thread.start()
with temporary_dir() as artifact_root:
artifact_cache = RESTfulArtifactCache(MockLogger(), artifact_root,
'http://localhost:%d' % port)
self.do_test_artifact_cache(artifact_cache)
finally:
if httpd:
httpd.shutdown()
if httpd_thread:
httpd_thread.join()
def execute(self):
dist_dir = self._config.getdefault('pants_distdir')
target_base = '%s-%s' % (
self.target.provides.name, self.target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
expected_tgz = '%s.tar.gz' % target_base
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(dist_dir, expected_tgz)
chroot = Chroot(dist_dir, name=self.target.provides.name)
self.write_contents(chroot)
self.write_setup(chroot)
safe_rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
stdout, stderr = po.communicate()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in ''.join(stdout).splitlines():
print('stdout: %s' % line)
for line in ''.join(stderr).splitlines():
print('stderr: %s' % line)
return po.returncode
else:
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
safe_rmtree(setup_dir)
print('Wrote %s' % dist_tgz)
def execute(self):
config = Config.load()
distdir = config.getdefault('pants_distdir')
setup_dir = os.path.join(distdir, '%s-%s' % (
self.target.provides._name, self.target.provides._version))
chroot = Chroot(distdir, name=self.target.provides._name)
self.write_sources(chroot)
self.write_setup(chroot)
if os.path.exists(setup_dir):
import shutil
shutil.rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
po.wait()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in po.stdout.read().splitlines():
print('stdout: %s' % line)
for line in po.stderr.read().splitlines():
print('stderr: %s' % line)
return po.returncode
expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name, self.target.provides._version)
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(distdir, expected_tgz)
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
print('Wrote %s' % dist_tgz)
safe_rmtree(setup_dir)
def test_upload(self):
users = {NATIVE_USER: {'password': NATIVE_PASSWORD}}
indices = {NATIVE_USER + '/index': {}}
with TestServer(users=users, indices=indices) as devpi:
devpi.use(NATIVE_USER, 'index')
devpi.login(NATIVE_USER, NATIVE_PASSWORD)
with pushd(SOURCE_DIR):
devpi.upload(path=None, with_docs=True)
def doc_present(version=PACKAGE_VERSION):
return requests.get(
devpi.server_url + "/{}/index/test-package/{}/+d/index.html".format(NATIVE_USER, version),
).status_code == 200,
wait_until(doc_present, maxloop=300)
self.assertTrue(doc_present('+latest'))
self.assertTrue(doc_present('+stable'))
def run(self, binary=None, interpreter_args=[], args=[], extra_deps=[], with_chroot=False, kill_orphans=False):
"""
Run the PythonEnvironment in an interpreter in a subprocess.
"""
cmdline = self.cmdline(binary, interpreter_args, args)
path = self.path(extras=extra_deps)
with pushd(self._dir.path() if with_chroot else os.getcwd()):
with environment_as(PYTHONPATH=":".join(path)):
PythonLauncher.debug("With PYTHONPATH=%s, executing %s" % (":".join(path), " ".join(cmdline)))
# Spawn in a new session so we can cleanup any orphans
po = subprocess.Popen(cmdline, preexec_fn=os.setsid)
rv = -1
try:
rv = po.wait()
finally:
if kill_orphans and rv:
self._reap_orphans(po.pid)
return rv
def test_upload(self):
users = {NATIVE_USER: {'password': NATIVE_PASSWORD}}
indices = {NATIVE_USER + '/index': {}}
with TestServer(users=users, indices=indices) as devpi:
devpi.use(NATIVE_USER, 'index')
devpi.login(NATIVE_USER, NATIVE_PASSWORD)
with pushd(SOURCE_DIR):
devpi.upload(path=None, with_docs=True)
def doc_present(version=PACKAGE_VERSION):
return requests.get(
devpi.server_url +
"/{}/index/test-package/{}/+d/index.html".format(
NATIVE_USER, version), ).status_code == 200,
wait_until(doc_present, maxloop=300)
self.assertTrue(doc_present('+latest'))
self.assertTrue(doc_present('+stable'))
def assert_pex_args_shebang(shebang):
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
)
""")
with temporary_content({'setup.py': setup_py}) as project_dir:
with pushd(project_dir):
assert subprocess.check_call(
[sys.executable, 'setup.py', 'bdist_pex',
'--pex-args=--python-shebang="%(shebang)s"' %
dict(shebang=shebang)]) == 0
with open(os.path.join(project_dir, 'dist',
'my_app-0.0.0.pex'), 'rb') as fp:
assert fp.readline().decode().rstrip() == shebang
def build_egg(self, egg_root, target):
"""Build an egg containing the files at egg_root for the specified target.
There must be an egg_root/setup.py file."""
# TODO(Brian Wickman): Do a sanity check somewhere to ensure that
# setuptools is on the path?
args = [sys.executable, "setup.py", "bdist_egg", "--dist-dir=dist", "--bdist-dir=build.%s" % target.name]
with pushd(egg_root):
with environment_as(PYTHONPATH=":".join(sys.path)):
po = subprocess.Popen(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
rv = po.wait()
eggs = os.path.abspath(os.path.join("dist", "*.egg"))
eggs = glob.glob(eggs)
if rv != 0 or len(eggs) != 1:
comm = po.communicate()
print("egg generation failed (return value=%d, num eggs=%d)" % (rv, len(eggs)), file=sys.stderr)
print("STDOUT", file=sys.stderr)
print(comm[0], file=sys.stderr)
print("STDERR", file=sys.stderr)
print(comm[1], file=sys.stderr)
raise EggBuilder.EggBuildingException("Generation of eggs failed for target = %s" % target)
egg_path = eggs[0]
return egg_path
def assert_pex_args_shebang(shebang):
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
)
""")
with temporary_content({'setup.py': setup_py}) as project_dir:
with pushd(project_dir):
assert subprocess.check_call([
sys.executable, 'setup.py', 'bdist_pex',
'--pex-args=--python-shebang="%(shebang)s"' %
dict(shebang=shebang)
]) == 0
with open(os.path.join(project_dir, 'dist', 'my_app-0.0.0.pex'),
'rb') as fp:
assert fp.readline().decode().rstrip() == shebang
def test_branch(tmpdir, repo):
with branch('a', repo=repo):
with pushd(tmpdir.strpath):
with open('test') as f:
assert f.read() == first_commit_file_content
def test_branch(tmpdir, repo):
with branch('a', repo=repo):
with pushd(tmpdir.strpath):
with open('test') as f:
assert f.read() == first_commit_file_content