def create_setup_py(self, target, dist_dir):
chroot = Chroot(dist_dir, name=target.provides.name)
dependency_calculator = self.DependencyCalculator(self.context.build_graph)
reduced_deps = dependency_calculator.reduced_dependencies(target)
self.write_contents(target, reduced_deps, chroot)
self.write_setup(target, reduced_deps, chroot)
target_base = '{}-{}'.format(target.provides.name, target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
safe_rmtree(setup_dir)
shutil.move(chroot.path(), setup_dir)
return setup_dir, reduced_deps
def create_setup_py(self, target, dist_dir):
chroot = Chroot(dist_dir, name=target.provides.name)
dependency_calculator = self.DependencyCalculator(self.context.build_graph)
reduced_deps = dependency_calculator.reduced_dependencies(target)
self.write_contents(target, reduced_deps, chroot)
self.write_setup(target, reduced_deps, chroot)
target_base = '{}-{}'.format(target.provides.name, target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
safe_rmtree(setup_dir)
shutil.move(chroot.path(), setup_dir)
return setup_dir, reduced_deps
def execute(self):
config = Config.load()
distdir = config.getdefault('pants_distdir')
setup_dir = os.path.join(
distdir, '%s-%s' %
(self.target.provides._name, self.target.provides._version))
chroot = Chroot(distdir, name=self.target.provides._name)
self.write_sources(chroot)
self.write_setup(chroot)
if os.path.exists(setup_dir):
import shutil
shutil.rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run
or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
po.wait()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in po.stdout.read().splitlines():
print('stdout: %s' % line)
for line in po.stderr.read().splitlines():
print('stderr: %s' % line)
return po.returncode
expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name,
self.target.provides._version)
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(distdir, expected_tgz)
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
print('Wrote %s' % dist_tgz)
safe_rmtree(setup_dir)
def execute(self):
dist_dir = self._config.getdefault('pants_distdir')
target_base = '%s-%s' % (
self.target.provides.name, self.target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
expected_tgz = '%s.tar.gz' % target_base
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(dist_dir, expected_tgz)
chroot = Chroot(dist_dir, name=self.target.provides.name)
self.write_contents(chroot)
self.write_setup(chroot)
safe_rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
stdout, stderr = po.communicate()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in ''.join(stdout).splitlines():
print('stdout: %s' % line)
for line in ''.join(stderr).splitlines():
print('stderr: %s' % line)
return po.returncode
else:
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
safe_rmtree(setup_dir)
print('Wrote %s' % dist_tgz)
def execute(self):
config = Config.load()
distdir = config.getdefault('pants_distdir')
setup_dir = os.path.join(distdir, '%s-%s' % (
self.target.provides._name, self.target.provides._version))
chroot = Chroot(distdir, name=self.target.provides._name)
self.write_sources(chroot)
self.write_setup(chroot)
if os.path.exists(setup_dir):
import shutil
shutil.rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
with pushd(setup_dir):
cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist')
print('Running "%s" in %s' % (cmd, setup_dir))
extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
po = subprocess.Popen(cmd, shell=True, **extra_args)
po.wait()
if self.options.run:
print('Ran %s' % cmd)
print('Output in %s' % setup_dir)
return po.returncode
elif po.returncode != 0:
print('Failed to run %s!' % cmd)
for line in po.stdout.read().splitlines():
print('stdout: %s' % line)
for line in po.stderr.read().splitlines():
print('stderr: %s' % line)
return po.returncode
expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name, self.target.provides._version)
expected_target = os.path.join(setup_dir, 'dist', expected_tgz)
dist_tgz = os.path.join(distdir, expected_tgz)
if not os.path.exists(expected_target):
print('Could not find expected target %s!' % expected_target)
sys.exit(1)
safe_delete(dist_tgz)
os.rename(expected_target, dist_tgz)
print('Wrote %s' % dist_tgz)
safe_rmtree(setup_dir)
def run_one(self, target):
dist_dir = self._config.getdefault('pants_distdir')
chroot = Chroot(dist_dir, name=target.provides.name)
self.write_contents(target, chroot)
self.write_setup(target, chroot)
target_base = '%s-%s' % (target.provides.name, target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
safe_rmtree(setup_dir)
shutil.move(chroot.path(), setup_dir)
if not self.old_options.run:
print('Running packager against %s' % setup_dir)
setup_runner = Packager(setup_dir)
tgz_name = os.path.basename(setup_runner.sdist())
print('Writing %s' % os.path.join(dist_dir, tgz_name))
shutil.move(setup_runner.sdist(), os.path.join(dist_dir, tgz_name))
safe_rmtree(setup_dir)
else:
print('Running %s against %s' % (self.old_options.run, setup_dir))
setup_runner = SetupPyRunner(setup_dir, self.old_options.run)
setup_runner.run()
def run_one(self, target):
dist_dir = self._config.getdefault('pants_distdir')
chroot = Chroot(dist_dir, name=target.provides.name)
self.write_contents(target, chroot)
self.write_setup(target, chroot)
target_base = '%s-%s' % (target.provides.name, target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
safe_rmtree(setup_dir)
os.rename(chroot.path(), setup_dir)
if not self.options.run:
print('Running packager against %s' % setup_dir)
setup_runner = Packager(setup_dir)
tgz_name = os.path.basename(setup_runner.sdist())
print('Writing %s' % os.path.join(dist_dir, tgz_name))
os.rename(setup_runner.sdist(), os.path.join(dist_dir, tgz_name))
safe_rmtree(setup_dir)
else:
print('Running %s against %s' % (self.options.run, setup_dir))
setup_runner = SetupPyRunner(setup_dir, self.options.run)
setup_runner.run()
def run_one(self, target):
dist_dir = self.get_options().pants_distdir
chroot = Chroot(dist_dir, name=target.provides.name)
self.write_contents(target, chroot)
self.write_setup(target, chroot)
target_base = '%s-%s' % (target.provides.name, target.provides.version)
setup_dir = os.path.join(dist_dir, target_base)
safe_rmtree(setup_dir)
shutil.move(chroot.path(), setup_dir)
if not self._run:
self.context.log.info('Running packager against %s' % setup_dir)
setup_runner = Packager(setup_dir)
tgz_name = os.path.basename(setup_runner.sdist())
self.context.log.info('Writing %s' % os.path.join(dist_dir, tgz_name))
shutil.move(setup_runner.sdist(), os.path.join(dist_dir, tgz_name))
safe_rmtree(setup_dir)
else:
self.context.log.info('Running %s against %s' % (self._run, setup_dir))
setup_runner = SetupPyRunner(setup_dir, self._run)
setup_runner.run()
def __init__(self, path=None):
self._chroot = Chroot(path or tempfile.mkdtemp())
self._pex_info = PexInfo.default()
self._frozen = False
self._logger = logging.getLogger(__name__)
class PEXBuilder(object):
class InvalidDependency(Exception):
pass
class InvalidExecutableSpecification(Exception):
pass
DEPENDENCY_DIR = ".deps"
BOOTSTRAP_DIR = ".bootstrap"
def __init__(self, path=None):
self._chroot = Chroot(path or tempfile.mkdtemp())
self._pex_info = PexInfo.default()
self._frozen = False
self._logger = logging.getLogger(__name__)
def chroot(self):
return self._chroot
def path(self):
return self.chroot().path()
def info(self):
return self._pex_info
def add_source(self, filename, env_filename):
self._chroot.link(filename, env_filename, "source")
if filename.endswith('.py'):
env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc'
with open(filename) as fp:
pyc_object = CodeMarshaller.from_py(fp.read(), env_filename)
self._chroot.write(pyc_object.to_pyc(), env_filename_pyc, 'source')
def add_resource(self, filename, env_filename):
self._chroot.link(filename, env_filename, "resource")
def add_requirement(self, req, dynamic=False, repo=None):
self._pex_info.add_requirement(req, repo=repo, dynamic=dynamic)
def add_dependency_file(self, filename, env_filename):
# TODO(wickman) This is broken. The build cache abstraction just breaks down here.
if filename.endswith('.egg'):
self.add_egg(filename)
else:
self._chroot.link(
filename, os.path.join(PEXBuilder.DEPENDENCY_DIR,
env_filename))
def add_egg(self, egg):
"""
helper for add_distribution
"""
metadata = EggMetadata(zipimporter(egg))
dist = Distribution.from_filename(egg, metadata)
self.add_distribution(dist)
self.add_requirement(dist.as_requirement(), dynamic=False, repo=None)
def add_distribution(self, dist):
if not dist.location.endswith('.egg'):
raise PEXBuilder.InvalidDependency(
'Non-egg dependencies not yet supported.')
self._chroot.link(
dist.location,
os.path.join(PEXBuilder.DEPENDENCY_DIR,
os.path.basename(dist.location)))
def set_executable(self, filename, env_filename=None):
if env_filename is None:
env_filename = os.path.basename(filename)
if self._chroot.get("executable"):
raise PEXBuilder.InvalidExecutableSpecification(
"Setting executable on a PEXBuilder that already has one!")
self._chroot.link(filename, env_filename, "executable")
entry_point = env_filename
entry_point.replace(os.path.sep, '.')
self._pex_info.entry_point = entry_point.rpartition('.')[0]
def _prepare_inits(self):
relative_digest = self._chroot.get("source")
init_digest = set()
for path in relative_digest:
split_path = path.split(os.path.sep)
for k in range(1, len(split_path)):
sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py'])
if sub_path not in relative_digest and sub_path not in init_digest:
self._chroot.touch(sub_path)
init_digest.add(sub_path)
def _prepare_manifest(self):
self._chroot.write(self._pex_info.dump().encode('utf-8'),
PexInfo.PATH,
label='manifest')
def _prepare_main(self):
self._chroot.write(BOOTSTRAP_ENVIRONMENT, '__main__.py', label='main')
def _prepare_bootstrap(self):
"""
Write enough of distribute into the .pex .bootstrap directory so that
we can be fully self-contained.
"""
bare_env = pkg_resources.Environment()
distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24')
distribute_dist = None
for dist in DistributionHelper.all_distributions(sys.path):
if dist in distribute_req and bare_env.can_add(dist):
distribute_dist = dist
break
else:
raise DistributionNotFound('Could not find distribute!')
for fn, content in DistributionHelper.walk_data(distribute_dist):
if fn.startswith('pkg_resources.py') or fn.startswith(
'setuptools'):
self._chroot.write(content,
os.path.join(self.BOOTSTRAP_DIR,
fn), 'resource')
libraries = ('twitter.common.dirutil', 'twitter.common.collections',
'twitter.common.contextutil', 'twitter.common.lang',
'twitter.common.python', 'twitter.common.python.http',
'twitter.common.quantity')
for name in libraries:
dirname = name.replace('.', '/')
provider = pkg_resources.get_provider(name)
if not isinstance(provider, pkg_resources.DefaultProvider):
mod = __import__(name, fromlist=['wutttt'])
provider = pkg_resources.ZipProvider(mod)
for fn in provider.resource_listdir(''):
if fn.endswith('.py'):
self._chroot.write(
provider.get_resource_string(name, fn),
os.path.join(self.BOOTSTRAP_DIR, dirname, fn),
'resource')
for initdir in ('twitter', 'twitter/common'):
self._chroot.write(
b"__import__('pkg_resources').declare_namespace(__name__)",
os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'),
'resource')
def freeze(self):
if self._frozen:
return
self._prepare_inits()
self._prepare_manifest()
self._prepare_bootstrap()
self._prepare_main()
self._frozen = True
def build(self, filename):
self.freeze()
try:
os.unlink(filename + '~')
self._logger.warn(
'Previous binary unexpectedly exists, cleaning: %s' %
(filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
# TODO(wickman) Make this tunable
pexfile.write(
Compatibility.to_bytes('%s\n' %
PythonIdentity.get().hashbang()))
self._chroot.zip(filename + '~', mode='a')
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False self._logger = logging.getLogger(__name__)
class PEXBuilder(object):
class InvalidDependency(Exception): pass
class InvalidExecutableSpecification(Exception): pass
DEPENDENCY_DIR = ".deps"
BOOTSTRAP_DIR = ".bootstrap"
def __init__(self, path=None):
self._chroot = Chroot(path or tempfile.mkdtemp())
self._pex_info = PexInfo.default()
self._frozen = False
self._logger = logging.getLogger(__name__)
def chroot(self):
return self._chroot
def path(self):
return self.chroot().path()
def info(self):
return self._pex_info
def add_source(self, filename, env_filename):
self._chroot.link(filename, env_filename, "source")
if filename.endswith('.py'):
env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc'
with open(filename) as fp:
pyc_object = CodeMarshaller.from_py(fp.read(), env_filename)
self._chroot.write(pyc_object.to_pyc(), env_filename_pyc, 'source')
def add_resource(self, filename, env_filename):
self._chroot.link(filename, env_filename, "resource")
def add_requirement(self, req, dynamic=False, repo=None):
self._pex_info.add_requirement(req, repo=repo, dynamic=dynamic)
def add_dependency_file(self, filename, env_filename):
# TODO(wickman) This is broken. The build cache abstraction just breaks down here.
if filename.endswith('.egg'):
self.add_egg(filename)
else:
self._chroot.link(filename, os.path.join(PEXBuilder.DEPENDENCY_DIR, env_filename))
def add_egg(self, egg):
"""
helper for add_distribution
"""
metadata = EggMetadata(zipimporter(egg))
dist = Distribution.from_filename(egg, metadata)
self.add_distribution(dist)
self.add_requirement(dist.as_requirement(), dynamic=False, repo=None)
def add_distribution(self, dist):
if not dist.location.endswith('.egg'):
raise PEXBuilder.InvalidDependency('Non-egg dependencies not yet supported.')
self._chroot.link(dist.location,
os.path.join(PEXBuilder.DEPENDENCY_DIR, os.path.basename(dist.location)))
def set_executable(self, filename, env_filename=None):
if env_filename is None:
env_filename = os.path.basename(filename)
if self._chroot.get("executable"):
raise PEXBuilder.InvalidExecutableSpecification(
"Setting executable on a PEXBuilder that already has one!")
self._chroot.link(filename, env_filename, "executable")
entry_point = env_filename
entry_point.replace(os.path.sep, '.')
self._pex_info.entry_point = entry_point.rpartition('.')[0]
def _prepare_inits(self):
relative_digest = self._chroot.get("source")
init_digest = set()
for path in relative_digest:
split_path = path.split(os.path.sep)
for k in range(1, len(split_path)):
sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py'])
if sub_path not in relative_digest and sub_path not in init_digest:
self._chroot.touch(sub_path)
init_digest.add(sub_path)
def _prepare_manifest(self):
self._chroot.write(self._pex_info.dump().encode('utf-8'), PexInfo.PATH, label='manifest')
def _prepare_main(self):
self._chroot.write(BOOTSTRAP_ENVIRONMENT, '__main__.py', label='main')
def _prepare_bootstrap(self):
"""
Write enough of distribute into the .pex .bootstrap directory so that
we can be fully self-contained.
"""
bare_env = pkg_resources.Environment()
distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24')
distribute_dist = None
for dist in DistributionHelper.all_distributions(sys.path):
if dist in distribute_req and bare_env.can_add(dist):
distribute_dist = dist
break
else:
raise DistributionNotFound('Could not find distribute!')
for fn, content in DistributionHelper.walk_data(distribute_dist):
if fn.startswith('pkg_resources.py') or fn.startswith('setuptools'):
self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource')
libraries = (
'twitter.common.dirutil',
'twitter.common.collections',
'twitter.common.contextutil',
'twitter.common.lang',
'twitter.common.python',
'twitter.common.python.http',
'twitter.common.quantity'
)
for name in libraries:
dirname = name.replace('.', '/')
provider = pkg_resources.get_provider(name)
if not isinstance(provider, pkg_resources.DefaultProvider):
mod = __import__(name, fromlist=['wutttt'])
provider = pkg_resources.ZipProvider(mod)
for fn in provider.resource_listdir(''):
if fn.endswith('.py'):
self._chroot.write(provider.get_resource_string(name, fn),
os.path.join(self.BOOTSTRAP_DIR, dirname, fn), 'resource')
for initdir in ('twitter', 'twitter/common'):
self._chroot.write(
b"__import__('pkg_resources').declare_namespace(__name__)",
os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'),
'resource')
def freeze(self):
if self._frozen:
return
self._prepare_inits()
self._prepare_manifest()
self._prepare_bootstrap()
self._prepare_main()
self._frozen = True
def build(self, filename):
self.freeze()
try:
os.unlink(filename + '~')
self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~'))
except OSError:
# The expectation is that the file does not exist, so continue
pass
with open(filename + '~', 'ab') as pexfile:
assert os.path.getsize(pexfile.name) == 0
# TODO(wickman) Make this tunable
pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang()))
self._chroot.zip(filename + '~', mode='a')
if os.path.exists(filename):
os.unlink(filename)
os.rename(filename + '~', filename)
chmod_plus_x(filename)
def __init__(self, path): self._chroot = Chroot(path)
class PythonEnvironment(object):
class InvalidDependency(Exception): pass
class InvalidExecutableSpecification(Exception): pass
DEPENDENCY_DIR = ".deps"
MAIN = """
import os
import sys
from twitter.common.python import PythonLauncher
__entry_point__ = None
if locals().has_key('__file__') and __file__ is not None:
__entry_point__ = os.path.dirname(__file__)
elif locals().has_key('__loader__'):
from zipimport import zipimporter
from pkgutil import ImpLoader
if isinstance(__loader__, zipimporter):
__entry_point__ = __loader__.archive
elif isinstance(__loader__, ImpLoader):
__entry_point__ = os.path.dirname(__loader__.get_filename())
if __entry_point__ is not None:
PythonLauncher(__entry_point__).execute()
else:
print >> sys.stderr, "Could not launch Python executable!"
sys.exit(2)
"""
def __init__(self, path):
self._chroot = Chroot(path)
def chroot(self):
return self._chroot
def path(self):
return self.chroot().path()
def add_source(self, filename, env_filename):
self._chroot.link(filename, env_filename, "source")
def add_resource(self, filename, env_filename):
self._chroot.link(filename, env_filename, "resource")
def add_dependency(self, dependency):
added_files = set()
if not isinstance(dependency, PythonDependency):
raise PythonEnvironment.InvalidDependency(
"Input dependency (%s) is not a valid PythonDependency!" % repr(dependency))
for fn, content in dependency.files():
added_files.add(os.path.join(self.path(), PythonEnvironment.DEPENDENCY_DIR, fn))
self._chroot.write(content, os.path.join(PythonEnvironment.DEPENDENCY_DIR, fn), "dependency")
return (os.path.join(self.path(), PythonEnvironment.DEPENDENCY_DIR), added_files)
def add_dependency_file(self, filename, dep_filename):
self._chroot.link(filename, os.path.join(PythonEnvironment.DEPENDENCY_DIR, dep_filename),
"dependency")
def set_executable(self, filename, env_filename=None):
if env_filename is None:
env_filename = os.path.basename(filename)
self._chroot.link(filename, env_filename, "executable")
def executable(self):
"""
Return the executable target of this environment if one is specified. If not
specified, return None.
"""
exe = self._chroot.get("executable")
if len(exe) > 1:
raise PythonEnvironment.InvalidExecutableSpecification(
"Expected one or zero executables, but instead got executables = %s" % repr(exe))
if len(exe) == 1:
return exe.copy().pop()
def _prepare_inits(self):
relative_digest = self._chroot.get("source")
init_digest = set()
for path in relative_digest:
split_path = path.split(os.path.sep)
for k in range(1, len(split_path)):
sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py'])
if sub_path not in relative_digest and sub_path not in init_digest:
self._chroot.touch(sub_path)
init_digest.add(sub_path)
def _script(self):
"""
Returns the module-form of the entry point or None if it's not there.
"""
if self.executable():
source = self.executable()
source.replace(os.path.sep, '.')
source = source.rpartition('.')[0]
return source
def _manifest(self):
manifest = {}
script = self._script()
if script:
manifest.update({'entry': script})
return json.dumps(manifest)
def _prepare_manifest(self):
self._chroot.write(self._manifest(), 'PEX-INFO', label='manifest')
def _prepare_main(self):
self._chroot.write(self.MAIN, '__main__.py', label='main')
def freeze(self):
self._prepare_inits()
self._prepare_manifest()
self._prepare_main()
def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False
