def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master'])
with safe_open(self.readme_file, 'w') as readme:
readme.write('Hello World.')
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def _gen_options_reference(self):
"""Generate the options reference rst doc."""
goals = gen_tasks_options_reference_data()
filtered_goals = []
omit_impl_regexps = [re.compile(r) for r in self.get_options().omit_impl_re]
for g in goals:
if any(r.match(t['impl']) for r in omit_impl_regexps for t in g.tasks):
continue
filtered_goals.append(g)
glopts = gen_glopts_reference_data()
# generate the .rst file
template = resource_string(__name__,
os.path.join(self._templates_dir, 'options_reference.mustache'))
filename = os.path.join(self._outdir, 'options_reference.rst')
self.context.log.info('Generating {}'.format(filename))
with safe_open(filename, 'wb') as outfile:
generator = Generator(template, goals=filtered_goals, glopts=glopts)
generator.write(outfile)
# generate the .html file
template = resource_string(__name__,
os.path.join(self._templates_dir, 'oref_html.mustache'))
filename = os.path.join(self._outdir, 'options_reference.html')
self.context.log.info('Generating {}'.format(filename))
with safe_open(filename, 'wb') as outfile:
generator = Generator(template, goals=filtered_goals, glopts=glopts)
generator.write(outfile)
def symlink_cachepath(ivy_home, inpath, symlink_dir, outpath):
"""Symlinks all paths listed in inpath that are under ivy_home into symlink_dir.
Preserves all other paths. Writes the resulting paths to outpath.
Returns a map of path -> symlink to that path.
"""
safe_mkdir(symlink_dir)
with safe_open(inpath, 'r') as infile:
paths = filter(None, infile.read().strip().split(os.pathsep))
new_paths = []
for path in paths:
if not path.startswith(ivy_home):
new_paths.append(path)
continue
symlink = os.path.join(symlink_dir, os.path.relpath(path, ivy_home))
try:
os.makedirs(os.path.dirname(symlink))
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Note: The try blocks cannot be combined. It may be that the dir exists but the link doesn't.
try:
os.symlink(path, symlink)
except OSError as e:
# We don't delete and recreate the symlink, as this may break concurrently executing code.
if e.errno != errno.EEXIST:
raise
new_paths.append(symlink)
with safe_open(outpath, 'w') as outfile:
outfile.write(':'.join(new_paths))
symlink_map = dict(zip(paths, new_paths))
return symlink_map
def test_go_test_unstyle(self):
with self.temporary_sourcedir() as srcdir:
lib_unstyle_relpath = 'src/go/libUnstyle'
lib_unstyle_dir = os.path.join(srcdir, lib_unstyle_relpath)
with safe_open(os.path.join(lib_unstyle_dir, 'unstyle.go'), 'w') as fp:
# NB: Go format violating indents below.
fp.write(dedent("""
package libUnstyle
func Speak() {
println("Hello from libUnstyle!")
println("Bye from libUnstyle!")
}
func Add(a int, b int) int {
return a + b
}
""").strip())
with safe_open(os.path.join(lib_unstyle_dir, 'BUILD'), 'w') as fp:
fp.write('go_library()')
args = ['compile', 'lint', lib_unstyle_dir]
pants_run = self.run_pants(args)
self.assert_failure(pants_run)
args = ['compile', 'lint', '--lint-go-skip', lib_unstyle_dir]
pants_run = self.run_pants(args)
self.assert_success(pants_run)
def _create_invalid_jax_ws_project(self):
with self.temporary_sourcedir() as srcdir:
with safe_open(os.path.join(srcdir, 'src/jax_ws/test/InvalidService.wsdl'), 'w') as fp:
fp.write(dedent("""
<definitions xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd"
xmlns:wsp="http://www.w3.org/ns/ws-policy"
xmlns:wsp1_2="http://schemas.xmlsoap.org/ws/2004/09/policy"
xmlns:wsam="http://www.w3.org/2007/05/addressing/metadata"
xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/"
xmlns:tns="http://example.com/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
xmlns="http://schemas.xmlsoap.org/wsdl/"
targetNamespace="http://example.com/"
name="InvalidServerImplService">
<types/>
<service name="InvalidServerImplService">
<port name="InvalidServerImplPort" binding="tns:InvalidServerImplPortBinding">
<soap:address location="http://127.0.0.1:9876/invalid"/>
</port>
</service>
</definitions>
""").strip())
with safe_open(os.path.join(srcdir, 'src/jax_ws/test/BUILD'), 'w') as fp:
fp.write(dedent("""
jax_ws_library(name='invalid-service',
sources = [
'InvalidService.wsdl',
],
dependencies = [],
)
""").strip())
yield srcdir
def known_commits(self):
with temporary_dir(root_dir=get_buildroot()) as worktree:
with safe_open(os.path.join(worktree, 'README'), 'w') as fp:
fp.write('Just a test tree.')
with initialize_repo(worktree=worktree, gitdir=os.path.join(worktree, '.git')) as git:
src_file = os.path.join(worktree, 'src/java/org/pantsbuild/Class.java')
with safe_open(src_file, 'w') as fp:
fp.write(dedent("""
package org.pantsbuild;
class Class {
static final int MEANING_OF_LIFE = 42;
}
"""))
src_build_file = os.path.join(worktree, 'src/java/org/pantsbuild/BUILD')
with safe_open(src_build_file, 'w') as fp:
fp.write("java_library(name='pantsbuild', sources=['Class.java'])")
git.add(src_file, src_build_file)
git.commit('Introduce Class.')
test_file = os.path.join(worktree, 'tests/java/org/pantsbuild/ClassTest.java')
with safe_open(test_file, 'w') as fp:
fp.write(dedent("""
package org.pantsbuild;
import org.junit.Assert;
import org.junit.Test;
public class ClassTest {
@Test public void test() {
Assert.assertEquals(42, Class.MEANING_OF_LIFE);
}
}
"""))
test_build_file = os.path.join(worktree, 'tests/java/org/pantsbuild/BUILD')
with safe_open(test_build_file, 'w') as fp:
fp.write(dedent("""
jar_library(name='junit', jars=[jar('junit', 'junit', '4.12')])
junit_tests(
name='pantsbuild',
sources=['ClassTest.java'],
dependencies=[
':junit',
'{}'
]
)
""").format(os.path.relpath(os.path.dirname(src_build_file), get_buildroot())))
git.add(test_file, test_build_file)
git.commit('Introduce ClassTest.')
yield
def _create_thrift_project(self):
with self.temporary_sourcedir() as srcdir:
with safe_open(os.path.join(srcdir, 'src/thrift/thrifttest/duck.thrift'), 'w') as fp:
fp.write(dedent("""
namespace go thrifttest.duck
struct Duck {
1: optional string quack,
}
service Feeder {
void feed(1:Duck duck),
}
""").strip())
with safe_open(os.path.join(srcdir, 'src/thrift/thrifttest/BUILD'), 'w') as fp:
fp.write(dedent("""
go_thrift_library(
name='fleem',
sources=['duck.thrift']
)
""").strip())
with safe_open(os.path.join(srcdir, 'src/go/usethrift/example.go'), 'w') as fp:
fp.write(dedent("""
package usethrift
import "thrifttest/duck"
func whatevs(f duck.Feeder) string {
d := duck.NewDuck()
f.Feed(d)
return d.GetQuack()
}
""").strip())
with safe_open(os.path.join(srcdir, 'src/go/usethrift/BUILD'), 'w') as fp:
fp.write(dedent("""
go_library(
dependencies=[
'{srcdir}/src/thrift/thrifttest:fleem'
]
)
""".format(srcdir=os.path.relpath(srcdir, get_buildroot()))).strip())
with safe_open(os.path.join(srcdir, '3rdparty/go/github.com/apache/thrift/BUILD'), 'w') as fp:
fp.write("go_remote_library(rev='0.9.3', pkg='lib/go/thrift')")
config = {
'gen.go-thrift': {
'thrift_import_target':
os.path.join(os.path.relpath(srcdir, get_buildroot()),
'3rdparty/go/github.com/apache/thrift:lib/go/thrift'),
'thrift_import': 'github.com/apache/thrift/lib/go/thrift'
}
}
yield srcdir, config
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
safe_mkdir(os.path.join(self.worktree, 'dir'))
with open(os.path.join(self.worktree, 'dir', 'f'), 'w') as f:
f.write("file in subdir")
# Make some symlinks
os.symlink('f', os.path.join(self.worktree, 'dir', 'relative-symlink'))
os.symlink('no-such-file', os.path.join(self.worktree, 'dir', 'relative-nonexistent'))
os.symlink('dir/f', os.path.join(self.worktree, 'dir', 'not-absolute\u2764'))
os.symlink('../README', os.path.join(self.worktree, 'dir', 'relative-dotdot'))
os.symlink('dir', os.path.join(self.worktree, 'link-to-dir'))
os.symlink('README/f', os.path.join(self.worktree, 'not-a-dir'))
os.symlink('loop1', os.path.join(self.worktree, 'loop2'))
os.symlink('loop2', os.path.join(self.worktree, 'loop1'))
subprocess.check_call(['git', 'add', 'README', 'dir', 'loop1', 'loop2',
'link-to-dir', 'not-a-dir'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
self.initial_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream-to', 'depot/master'])
with safe_open(self.readme_file, 'w') as readme:
readme.write('Hello World.\u2764'.encode('utf-8'))
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.current_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def symlink_cachepath(cls, ivy_cache_dir, inpath, symlink_dir, outpath, existing_symlink_map):
"""Symlinks all paths listed in inpath that are under ivy_cache_dir into symlink_dir.
If there is an existing symlink for a file under inpath, it is used rather than creating
a new symlink. Preserves all other paths. Writes the resulting paths to outpath.
Returns a map of path -> symlink to that path.
"""
safe_mkdir(symlink_dir)
# The ivy_cache_dir might itself be a symlink. In this case, ivy may return paths that
# reference the realpath of the .jar file after it is resolved in the cache dir. To handle
# this case, add both the symlink'ed path and the realpath to the jar to the symlink map.
real_ivy_cache_dir = os.path.realpath(ivy_cache_dir)
updated_symlink_map = OrderedDict()
with safe_open(inpath, 'r') as infile:
inpaths = filter(None, infile.read().strip().split(os.pathsep))
paths = OrderedSet()
for path in inpaths:
paths.add(path)
realpath = os.path.realpath(path)
if path != realpath:
paths.add(realpath)
if realpath.startswith(real_ivy_cache_dir):
paths.add(os.path.join(ivy_cache_dir, realpath[len(real_ivy_cache_dir)+1:]))
for path in paths:
if path.startswith(ivy_cache_dir):
updated_symlink_map[path] = os.path.join(symlink_dir, os.path.relpath(path, ivy_cache_dir))
elif path.startswith(real_ivy_cache_dir):
updated_symlink_map[path] = os.path.join(symlink_dir, os.path.relpath(path, real_ivy_cache_dir))
else:
# This path is outside the cache. We won't symlink it.
updated_symlink_map[path] = path
# Create symlinks for paths in the ivy cache dir that we haven't seen before.
new_symlinks = cls._find_new_symlinks(existing_symlink_map, updated_symlink_map)
for path, symlink in new_symlinks.iteritems():
if path == symlink:
# Skip paths that aren't going to be symlinked.
continue
safe_mkdir(os.path.dirname(symlink))
try:
os.symlink(path, symlink)
except OSError as e:
# We don't delete and recreate the symlink, as this may break concurrently executing code.
if e.errno != errno.EEXIST:
raise
# (re)create the classpath with all of the paths
with safe_open(outpath, 'w') as outfile:
outfile.write(':'.join(OrderedSet(updated_symlink_map.values())))
return dict(updated_symlink_map)
def post_fork_child(self, fingerprint, jvm_options, classpath, stdout, stderr):
"""Post-fork() child callback for ProcessManager.daemon_spawn()."""
java = SubprocessExecutor(self._distribution)
subproc = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=safe_open('/dev/null', 'r'),
stdout=safe_open(self._ng_stdout, 'w'),
stderr=safe_open(self._ng_stderr, 'w'),
close_fds=True)
self.write_pid(subproc.pid)
def assert_checker(self, relpath, contents, expected_code=0, expected_message=''):
with temporary_dir() as td:
with safe_open(os.path.join(td, relpath), 'w') as fp:
fp.write(contents)
args=['--root-dir={}'.format(td)]
for plugin_type in checker.plugins():
opts = {'skip': False, 'max_length': self._MAX_LENGTH, 'ignore': ['E111']}
args.append('--{}-options={}'.format(plugin_type.name(), json.dumps(opts)))
args.append(relpath)
with open(os.path.join(td, 'stdout'), 'w+') as stdout:
with open(os.path.join(td, 'stderr'), 'w+') as stderr:
with stdio_as(stdout_fd=stdout.fileno(), stderr_fd=stderr.fileno(), stdin_fd=-1):
with self.assertRaises(SystemExit) as error:
checker.main(args=args)
def read_stdio(fp):
fp.flush()
fp.seek(0)
return fp.read()
self.assertEqual(expected_code, error.exception.code,
'STDOUT:\n{}\nSTDERR:\n{}'.format(read_stdio(stdout),
read_stdio(stderr)))
self.assertEqual(expected_message, read_stdio(stdout).strip())
self.assertEqual('', read_stdio(stderr))
def test_support_url_multi(self):
"""Tests to make sure existing base urls function as expected."""
bootstrap_dir = '/tmp'
with temporary_dir() as invalid_local_files, temporary_dir() as valid_local_files:
binary_util = self._gen_binary_util(
baseurls=[
'BLATANTLY INVALID URL',
'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
invalid_local_files,
valid_local_files,
'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
],
bootstrap_dir=bootstrap_dir)
binary_request = binary_util._make_deprecated_binary_request(
supportdir='bin/protobuf',
version='2.4.1',
name='protoc')
binary_path = binary_request.get_download_path(binary_util._host_platform())
contents = b'proof'
with safe_open(os.path.join(valid_local_files, binary_path), 'wb') as fp:
fp.write(contents)
binary_path_abs = os.path.join(bootstrap_dir, binary_path)
self.assertEqual(os.path.realpath(binary_path_abs),
os.path.realpath(binary_util.select(binary_request)))
self.assertEqual(contents, self._read_file(binary_path_abs))
def execute(self):
pages = []
targets = self.context.targets()
for target in targets:
if isinstance(target, Page):
for wiki_artifact in target.payload.provides:
pages.append((target, wiki_artifact))
urls = list()
genmap = self.context.products.get('wiki_html')
for page, wiki_artifact in pages:
html_info = genmap.get((wiki_artifact, page))
if len(html_info) > 1:
raise TaskError('Unexpected resources for %s: %s' % (page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError('Unexpected resources for %s: %s' % (page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
page.address,
wiki_artifact.config['space'],
wiki_artifact.config['title'],
contents.read(),
# Default to none if not present in the hash.
parent=wiki_artifact.config.get('parent')
)
if url:
urls.append(url)
self.context.log.info('Published %s to %s' % (page, url))
if self.open and urls:
binary_util.ui_open(*urls)
def _await_nailgun_server(self, stdout, stderr):
nailgun_timeout_seconds = 5
max_socket_connect_attempts = 10
nailgun = None
port_parse_start = time.time()
with safe_open(self._ng_out, 'r') as ng_out:
while not nailgun:
started = ng_out.readline()
if started:
port = self._parse_nailgun_port(started)
nailgun = self._create_ngclient(port, stdout, stderr)
log.debug('Detected ng server up on port %d' % port)
elif time.time() - port_parse_start > nailgun_timeout_seconds:
raise NailgunClient.NailgunError('Failed to read ng output after'
' %s seconds' % nailgun_timeout_seconds)
attempt = 0
while nailgun:
sock = nailgun.try_connect()
if sock:
sock.close()
endpoint = self._get_nailgun_endpoint()
if endpoint:
log.debug('Connected to ng server with fingerprint %s pid: %d @ port: %d' % endpoint)
else:
raise NailgunClient.NailgunError('Failed to connect to ng server.')
return nailgun
elif attempt > max_socket_connect_attempts:
raise nailgun.NailgunError('Failed to connect to ng output after %d connect attempts'
% max_socket_connect_attempts)
attempt += 1
log.debug('Failed to connect on attempt %d' % attempt)
time.sleep(0.1)
def commit_contents_to_files(content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path), 'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'commit', '-m', 'change '+path])
return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
def test_code_syntax_error(self):
with safe_open(self.fullpath('BUILD.badsyntax'), 'w') as fp:
fp.write('java_library(name=if)')
build_file = self.create_buildfile('BUILD.badsyntax')
with self.assertRaises(SyntaxError) as e:
build_file.code()
self.assertEqual(build_file.full_path, e.exception.filename)
def checkstyle(self, sources, targets):
egroups = self.context.products.get_data('exclusives_groups')
etag = egroups.get_group_key_for_target(targets[0])
classpath = self.tool_classpath(self._checkstyle_bootstrap_key)
cp = egroups.get_classpath_for_group(etag)
classpath.extend(jar for conf, jar in cp if conf in self._confs)
args = [
'-c', self._configuration_file,
'-f', 'plain'
]
if self._properties:
properties_file = os.path.join(self.workdir, 'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
args.extend(['-p', properties_file])
# We've hit known cases of checkstyle command lines being too long for the system so we guard
# with Xargs since checkstyle does not accept, for example, @argfile style arguments.
def call(xargs):
return self.runjava(classpath=classpath, main=CHECKSTYLE_MAIN,
args=args + xargs, workunit_name='checkstyle')
checks = Xargs(call)
return checks.execute(sources)
def checkstyle(self, targets, sources):
runtime_classpaths = self.context.products.get_data('runtime_classpath')
union_classpath = OrderedSet(self.tool_classpath('checkstyle'))
for target in targets:
runtime_classpath = runtime_classpaths.get_for_targets(target.closure(bfs=True))
union_classpath.update(jar for conf, jar in runtime_classpath
if conf in self.get_options().confs)
args = [
'-c', self.get_options().configuration,
'-f', 'plain'
]
if self.get_options().properties:
properties_file = os.path.join(self.workdir, 'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self.get_options().properties.items():
pf.write('{key}={value}\n'.format(key=k, value=v))
args.extend(['-p', properties_file])
# We've hit known cases of checkstyle command lines being too long for the system so we guard
# with Xargs since checkstyle does not accept, for example, @argfile style arguments.
def call(xargs):
return self.runjava(classpath=union_classpath, main=self._CHECKSTYLE_MAIN,
jvm_options=self.get_options().jvm_options,
args=args + xargs, workunit_name='checkstyle')
checks = Xargs(call)
return checks.execute(sources)
def _await_socket(self, timeout):
"""Blocks for the nailgun subprocess to bind and emit a listening port in the nailgun stdout."""
with safe_open(self._ng_stdout, 'r') as ng_stdout:
start_time = time.time()
accumulated_stdout = ''
while 1:
readable, _, _ = select.select([ng_stdout], [], [], self._SELECT_WAIT)
if readable:
line = ng_stdout.readline() # TODO: address deadlock risk here.
try:
return self._NG_PORT_REGEX.match(line).group(1)
except AttributeError:
pass
accumulated_stdout += line
if (time.time() - start_time) > timeout:
stderr = read_file(self._ng_stderr)
raise NailgunClient.NailgunError(
'Failed to read nailgun output after {sec} seconds!\n'
'Stdout:\n{stdout}\nStderr:\n{stderr}'.format(
sec=timeout,
stdout=accumulated_stdout,
stderr=stderr,
)
)
def test_parse_test_info_invalid_dir(self):
with temporary_dir() as xml_dir:
with safe_open(os.path.join(xml_dir, 'subdir', 'TEST-c.xml'), 'w') as fp:
fp.write('<invalid></xml>')
tests_info = self.parse_test_info(xml_dir, self._raise_handler)
self.assertEqual({}, tests_info)
def _emit_targets(cls, workdir):
with safe_open(os.path.join(workdir, 'BUILD'), 'w') as f:
f.write(dedent("""
remote_sources(name='proto-7',
dest=java_protobuf_library,
sources_target=':external-source',
args=dict(
platform='java7',
),
)
remote_sources(name='proto-8',
dest=java_protobuf_library,
sources_target=':external-source',
args=dict(
platform='java8',
),
)
unpacked_jars(name='external-source',
libraries=[':external-source-jars'],
include_patterns=[
'com/squareup/testing/**/*.proto',
],
)
jar_library(name='external-source-jars',
jars=[
jar(org='com.squareup.testing.protolib', name='protolib-external-test', rev='0.0.2'),
],
)
"""))
return ['{}:proto-{}'.format(os.path.relpath(workdir, get_buildroot()), num) for num in (7, 8)]
def run_pants_with_workdir(self, command, workdir, config=None, stdin_data=None, extra_env=None,
**kwargs):
args = ['--no-pantsrc',
'--pants-workdir=' + workdir,
'--kill-nailguns',
'--print-exception-stacktrace']
if config:
config_data = config.copy()
ini = ConfigParser.ConfigParser(defaults=config_data.pop('DEFAULT', None))
for section, section_config in config_data.items():
ini.add_section(section)
for key, value in section_config.items():
ini.set(section, key, value)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
ini.write(fp)
args.append('--config-override=' + ini_file_name)
pants_script = os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)
pants_command = [pants_script] + args + command
env = os.environ.copy()
env.update(extra_env or {})
proc = subprocess.Popen(pants_command, env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
(stdout_data, stderr_data) = proc.communicate(stdin_data)
return PantsResult(pants_command, proc.returncode, stdout_data.decode("utf-8"),
stderr_data.decode("utf-8"), workdir)
def java_home_exe(self):
with distribution(executables=EXE('bin/java', version='1')) as jdk1_home:
with distribution(executables=EXE('bin/java', version='2')) as jdk2_home:
with temporary_dir() as tmpdir:
osx_java_home_exe = os.path.join(tmpdir, 'java_home')
with safe_open(osx_java_home_exe, 'w') as fp:
fp.write(textwrap.dedent("""
#!/bin/sh
echo '<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<array>
<dict>
<key>JVMHomePath</key>
<string>{jdk1_home}</string>
</dict>
<dict>
<key>JVMHomePath</key>
<string>{jdk2_home}</string>
</dict>
</array>
</plist>
'
""".format(jdk1_home=jdk1_home, jdk2_home=jdk2_home)).strip())
chmod_plus_x(osx_java_home_exe)
original_osx_java_home_exe = DistributionLocator._OSX_JAVA_HOME_EXE
DistributionLocator._OSX_JAVA_HOME_EXE = osx_java_home_exe
try:
yield jdk1_home, jdk2_home
finally:
DistributionLocator._OSX_JAVA_HOME_EXE = original_osx_java_home_exe
def select_binary(self, name):
"""Selects a binary matching the current os and architecture.
:param name: the name of the binary to fetch.
:raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version
and name could be found.
"""
# TODO(John Sirois): finish doc of the path structure expected under base_path
binary_path = BinaryUtil.select_binary_base_path(self._supportdir, self._version, name)
bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir))
bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
if not os.path.exists(bootstrapped_binary_path):
downloadpath = bootstrapped_binary_path + '~'
try:
with self.select_binary_stream(name) as stream:
with safe_open(downloadpath, 'wb') as bootstrapped_binary:
bootstrapped_binary.write(stream())
os.rename(downloadpath, bootstrapped_binary_path)
chmod_plus_x(bootstrapped_binary_path)
finally:
safe_delete(downloadpath)
logger.debug('Selected {binary} binary bootstrapped to: {path}'
.format(binary=name, path=bootstrapped_binary_path))
return bootstrapped_binary_path
def _await_socket(self, timeout):
"""Blocks for the nailgun subprocess to bind and emit a listening port in the nailgun stdout."""
with safe_open(self._ng_stdout, 'r') as ng_stdout:
start_time = time.time()
accumulated_stdout = ''
while 1:
# TODO: share the decreasing timeout logic here with NailgunProtocol.iter_chunks() by adding
# a method to pants.util.contextutil!
remaining_time = time.time() - (start_time + timeout)
if remaining_time > 0:
stderr = read_file(self._ng_stderr, binary_mode=True)
raise self.InitialNailgunConnectTimedOut(
timeout=timeout,
stdout=accumulated_stdout,
stderr=stderr,
)
readable, _, _ = select.select([ng_stdout], [], [], (-1 * remaining_time))
if readable:
line = ng_stdout.readline() # TODO: address deadlock risk here.
try:
return self._NG_PORT_REGEX.match(line).group(1)
except AttributeError:
pass
accumulated_stdout += line
def _download_zip(self, zip_url, dest_dir):
"""Downloads a zip file at the given URL into the given directory.
:param str zip_url: Full URL pointing to zip file.
:param str dest_dir: Absolute path of directory into which the unzipped contents
will be placed into, not including the zip directory itself.
"""
# TODO(jsirois): Wrap with workunits, progress meters, checksums.
self.context.log.info('Downloading {}...'.format(zip_url))
sess = requests.session()
sess.mount('file://', self.LocalFileAdapter())
res = sess.get(zip_url)
if not res.status_code == requests.codes.ok:
raise TaskError('Failed to download {} ({} error)'.format(zip_url, res.status_code))
with open_zip(BytesIO(res.content)) as zfile:
safe_mkdir(dest_dir)
for info in zfile.infolist():
if info.filename.endswith('/'):
# Skip directories.
continue
# Strip zip directory name from files.
filename = os.path.relpath(info.filename, get_basedir(info.filename))
f = safe_open(os.path.join(dest_dir, filename), 'w')
f.write(zfile.read(info))
f.close()
def run_pants_with_workdir(self, command, workdir, config=None, stdin_data=None, extra_env=None, **kwargs):
config = config.copy() if config else {}
# We add workdir to the DEFAULT section, and also ensure that it's emitted first.
default_section = config.pop('DEFAULT', {})
default_section['pants_workdir'] = '%s' % workdir
ini = ''
for section, section_config in [('DEFAULT', default_section)] + config.items():
ini += '\n[%s]\n' % section
for key, val in section_config.items():
ini += '%s: %s\n' % (key, val)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
fp.write(ini)
env = os.environ.copy()
env['PANTS_CONFIG_OVERRIDE'] = ini_file_name
env.update(extra_env or {})
pants_command = ([os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)] + command +
['--no-lock', '--kill-nailguns'])
proc = subprocess.Popen(pants_command, env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
(stdout_data, stderr_data) = proc.communicate(stdin_data)
return PantsResult(pants_command, proc.returncode, stdout_data, stderr_data)
def write(self, target, path, confs=None, extra_confs=None):
# TODO(John Sirois): a dict is used here to de-dup codegen targets which have both the original
# codegen target - say java_thrift_library - and the synthetic generated target (java_library)
# Consider reworking codegen tasks to add removal of the original codegen targets when rewriting
# the graph
dependencies = OrderedDict()
internal_codegen = {}
configurations = set(confs or [])
for dep in target_internal_dependencies(target):
jar = self._as_versioned_jar(dep)
dependencies[(jar.org, jar.name)] = self.internaldep(jar, dep)
if dep.is_codegen:
internal_codegen[jar.name] = jar.name
for jar in target.jar_dependencies:
if jar.rev:
dependencies[(jar.org, jar.name)] = self.jardep(jar)
configurations |= set(jar._configurations)
target_jar = self.internaldep(self._as_versioned_jar(target),
configurations=list(configurations))
target_jar = target_jar.extend(dependencies=dependencies.values())
template_kwargs = self.templateargs(target_jar, confs, extra_confs)
with safe_open(path, 'w') as output:
template = pkgutil.get_data(self.template_package_name, self.template_relpath)
Generator(template, **template_kwargs).write(output)
def run_pants_with_workdir(self, command, workdir, config=None, stdin_data=None, extra_env=None,
**kwargs):
config = config.copy() if config else {}
# We add workdir to the DEFAULT section, and also ensure that it's emitted first.
default_section = config.pop('DEFAULT', {})
default_section['pants_workdir'] = '%s' % workdir
ini = ''
for section, section_config in [('DEFAULT', default_section)] + config.items():
ini += '\n[%s]\n' % section
for key, val in section_config.items():
ini += '%s: %s\n' % (key, val)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
fp.write(ini)
env = os.environ.copy()
env.update(extra_env or {})
pants_script = os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)
pants_command = [pants_script,
'--kill-nailguns',
'--no-pantsrc',
'--config-override={0}'.format(ini_file_name),
'--print-exception-stacktrace'] + command
proc = subprocess.Popen(pants_command, env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
(stdout_data, stderr_data) = proc.communicate(stdin_data)
return PantsResult(pants_command, proc.returncode, stdout_data.decode("utf-8"),
stderr_data.decode("utf-8"))
def generate_build_files(self, targets):
goal_name = self.options_scope
flags = "--materialize"
if self.get_options().remote:
flags += " --remote"
template_header = dedent(
"""\
# Auto-generated by pants!
# To re-generate run: `pants {goal_name} {flags}`
"""
).format(goal_name=goal_name, flags=flags)
template_text = template_header + self.get_options().template
build_file_basename = "BUILD" + self.get_options().extension
targets_by_spec_path = defaultdict(set)
for target in targets:
targets_by_spec_path[target.address.spec_path].add(target)
for spec_path, targets in targets_by_spec_path.items():
rel_path = os.path.join(spec_path, build_file_basename)
result = self._create_template_data(rel_path, list(targets))
if result:
generator = Generator(template_text, target=result.data)
build_file_path = os.path.join(get_buildroot(), rel_path)
with safe_open(build_file_path, mode="w") as fp:
generator.write(stream=fp)
yield result
def test_pantsd_invalidation_pants_toml_file(self):
# Test tmp_pants_toml (--pants-config-files=$tmp_pants_toml)'s removal
tmp_pants_toml = os.path.abspath("testprojects/test_pants.toml")
# Create tmp_pants_toml file
with safe_open(tmp_pants_toml, "w") as f:
f.write("[DEFAULT]\n")
with self.pantsd_successful_run_context() as ctx:
ctx.runner([f"--pants-config-files={tmp_pants_toml}", "help"])
ctx.checker.assert_started()
time.sleep(5)
# Delete tmp_pants_toml
os.unlink(tmp_pants_toml)
time.sleep(10)
ctx.checker.assert_stopped()
def test_pantsd_invalidation_pants_ini_file(self):
# Test tmp_pants_ini (--pants-config-files=$tmp_pants_ini)'s removal
tmp_pants_ini = os.path.abspath("testprojects/test_pants.ini")
# Create tmp_pants_ini file
with safe_open(tmp_pants_ini, 'w') as f:
f.write("[DEFAULT]\n")
with self.pantsd_successful_run_context() as (pantsd_run, checker, _, _):
pantsd_run(['--pants-config-files={}'.format(tmp_pants_ini), 'help'])
checker.assert_started()
time.sleep(5)
# Delete tmp_pants_ini
os.unlink(tmp_pants_ini)
time.sleep(10)
checker.assert_stopped()
def generate_ivy(self, jar, version, publications):
template_relpath = os.path.join(_TEMPLATES_RELPATH, 'ivy.mustache')
template_text = pkgutil.get_data(__name__, template_relpath)
pubs = [TemplateData(name=None if p.name == jar.name else p.name,
classifier=p.classifier,
ext=None if p.ext == 'jar' else p.ext) for p in publications]
generator = Generator(template_text,
org=jar.org,
name=jar.name,
rev=version,
publications=pubs)
with safe_open(os.path.join(self.workdir, 'ivy.xml'), 'w') as ivyxml:
generator.write(ivyxml)
return ivyxml.name
def stage_artifact(tgt, jar, version, tag, changelog, confs=None, artifact_ext='',
extra_confs=None, classifier=''):
def path(name=None, suffix='', extension='jar'):
return self.artifact_path(jar, version, name=name, suffix=suffix, extension=extension,
artifact_ext=artifact_ext)
if self.publish_changelog:
with safe_open(path(suffix='-CHANGELOG', extension='txt'), 'wb') as changelog_file:
changelog_file.write(changelog.encode('utf-8'))
ivyxml = path(name='ivy', extension='xml')
IvyWriter(get_pushdb).write(tgt, ivyxml, confs=confs, extra_confs=extra_confs,
classifier=classifier)
PomWriter(get_pushdb, tag).write(tgt, path(extension='pom'), extra_confs=extra_confs,
classifier=classifier)
return ivyxml
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "configuration", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Configuration(name='c'), resolved)
# But success is cached.
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/c')))
def create_file(self,
relpath: str,
contents: str = "",
mode: str = "w") -> str:
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
self._invalidate_for(relpath)
return path
def _fetch_binary(self, name, binary_path):
bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir))
bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
if not os.path.exists(bootstrapped_binary_path):
downloadpath = bootstrapped_binary_path + '~'
try:
with self._select_binary_stream(name, binary_path) as stream:
with safe_open(downloadpath, 'wb') as bootstrapped_binary:
bootstrapped_binary.write(stream())
os.rename(downloadpath, bootstrapped_binary_path)
chmod_plus_x(bootstrapped_binary_path)
finally:
safe_delete(downloadpath)
logger.debug('Selected {binary} binary bootstrapped to: {path}'
.format(binary=name, path=bootstrapped_binary_path))
return bootstrapped_binary_path
def tree(self) -> Iterator[Tuple[str, str]]:
# root/
# a/
# b/
# 1
# 2
# 2 -> root/a/b/2
# b -> root/a/b
with temporary_dir() as root:
with safe_open(os.path.join(root, "a", "b", "1"), "wb") as fp:
fp.write(b"1")
touch(os.path.join(root, "a", "b", "2"))
os.symlink(os.path.join(root, "a", "b", "2"),
os.path.join(root, "a", "2"))
os.symlink(os.path.join(root, "a", "b"), os.path.join(root, "b"))
with temporary_dir() as dst:
yield root, dst
def report(self, targets, tests, tests_failed_exception=None):
if tests_failed_exception:
self._context.log.warn('Test failed: {0}'.format(str(tests_failed_exception)))
if self._coverage_force:
self._context.log.warn('Generating report even though tests failed')
else:
return
args = [
'report',
'-in', self._coverage_metadata_file,
'-in', self._coverage_file,
'-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self._test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
args.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
args.extend(['-r', 'txt',
'-Dreport.txt.out.file={0}'.format(self._coverage_console_file)] + sorting)
args.extend(['-r', 'xml', '-Dreport.xml.out.file={0}'.format(self._coverage_xml_file)])
args.extend(['-r', 'html',
'-Dreport.html.out.file={0}'.format(self._coverage_html_file),
'-Dreport.out.encoding=UTF-8'] + sorting)
main = 'emma'
result = execute_java(classpath=self._emma_classpath,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-report')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to generate code coverage reports'".format(main, result))
with safe_open(self._coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self._coverage_open:
binary_util.ui_open(self._coverage_html_file)
def test_no_address_no_family(self):
spec = SingleAddress('a/c', None)
# Should fail: does not exist.
with self.assertRaises(ResolveError):
self.resolve(spec)
# Exists on disk, but not yet in memory.
# NB: Graph invalidation not yet implemented.
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
with self.assertRaises(ResolveError):
self.resolve(spec)
# Success.
self.scheduler.product_graph.clear()
self.assertEqual(Struct(name='c'), self.resolve(spec).struct)
def test_invalidate_build_file_changed(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
with safe_open(build_file, 'w+') as fp:
fp.write('{"type_alias": "configuration", "name": "c"}')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
resolved = self.address_mapper.resolve(Address.parse('a/b:c'))
self.assertEqual(Configuration(name='c'), resolved)
# But success is cached.
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b:c')))
def prepare_resources(self, target, chroot):
for service, impls in target.services.items():
if impls:
service_provider_configuration_file = os.path.join(
chroot, self.service_info_path(service))
# NB: provider configuration files must be UTF-8 encoded, see the mini-spec:
# https://docs.oracle.com/javase/6/docs/api/java/util/ServiceLoader.html
with safe_open(service_provider_configuration_file,
'wb') as fp:
def write_line(line):
fp.write((line + '\n').encode('utf-8'))
write_line('# Generated from pants target {}'.format(
target.address.spec))
for impl in impls:
write_line(impl)
def _log_exception(self, msg):
if self._workdir:
try:
output_path = os.path.join(self._workdir, 'logs',
'exceptions.log')
with safe_open(output_path, 'a') as exception_log:
exception_log.write('timestamp: {}\n'.format(
datetime.datetime.now().isoformat()))
exception_log.write('args: {}\n'.format(sys.argv))
exception_log.write('pid: {}\n'.format(os.getpid()))
exception_log.write(msg)
exception_log.write('\n')
except Exception as e:
# This is all error recovery logic so we catch all exceptions from the logic above because
# we don't want to hide the original error.
logger.error(
'Problem logging original exception: {}'.format(e))
def tree(self) -> Iterator[Tuple[str, str]]:
# root/
# a/
# b/
# 1
# 2
# 2 -> root/a/b/2
# b -> root/a/b
with temporary_dir() as root:
with safe_open(os.path.join(root, 'a', 'b', '1'), 'wb') as fp:
fp.write(b'1')
touch(os.path.join(root, 'a', 'b', '2'))
os.symlink(os.path.join(root, 'a', 'b', '2'),
os.path.join(root, 'a', '2'))
os.symlink(os.path.join(root, 'a', 'b'), os.path.join(root, 'b'))
with temporary_dir() as dst:
yield root, dst
def _await_nailgun_server(self, stdout, stderr):
nailgun_timeout_seconds = 5
max_socket_connect_attempts = 10
nailgun = None
port_parse_start = time.time()
with safe_open(self._ng_out, 'r') as ng_out:
while not nailgun:
started = ng_out.readline()
if started.find(
'Listening for transport dt_socket at address:') >= 0:
nailgun_timeout_seconds = 60
log.warn(
'Timeout extended to {timeout} seconds for debugger to attach to ng server.'
.format(timeout=nailgun_timeout_seconds))
started = ng_out.readline()
if started:
port = self._parse_nailgun_port(started)
nailgun = self._create_ngclient(port, stdout, stderr)
log.debug('Detected ng server up on port %d' % port)
elif time.time() - port_parse_start > nailgun_timeout_seconds:
raise NailgunClient.NailgunError(
'Failed to read ng output after'
' %s seconds' % nailgun_timeout_seconds)
attempt = 0
while nailgun:
sock = nailgun.try_connect()
if sock:
sock.close()
endpoint = self._get_nailgun_endpoint()
if endpoint:
log.debug(
'Connected to ng server launched with %s fingerprint %s pid: %d @ port: %d'
% endpoint)
else:
raise NailgunClient.NailgunError(
'Failed to connect to ng server.')
return nailgun
elif attempt > max_socket_connect_attempts:
raise nailgun.NailgunError(
'Failed to connect to ng output after %d connect attempts'
% max_socket_connect_attempts)
attempt += 1
log.debug('Failed to connect on attempt %d' % attempt)
time.sleep(0.1)
def test_vcs_versioning(tag_regex, tag, expected_version, tmp_path: Path,
rule_runner: RuleRunner) -> None:
worktree = tmp_path / "worktree"
gitdir = worktree / ".git"
with safe_open(worktree / "README", "w") as f:
f.write("dummy content")
tag_regex_field = "" if tag_regex is None else f"tag_regex='{tag_regex}',"
rule_runner.write_files({
"src/python/BUILD":
textwrap.dedent(f"""
vcs_version(
name="version",
{tag_regex_field}
generate_to="src/python/_version.py",
template ='version = "{{version}}"'
)
""")
})
address = Address("src/python", target_name="version")
tgt = rule_runner.get_target(address)
with environment_as(
GIT_DIR=str(gitdir),
GIT_WORK_TREE=str(worktree),
GIT_CONFIG_GLOBAL="/dev/null",
):
subprocess.check_call(["git", "init"])
subprocess.check_call(
["git", "config", "user.email", "*****@*****.**"])
subprocess.check_call(["git", "config", "user.name", "Your Name"])
subprocess.check_call(["git", "add", "."])
subprocess.check_call(["git", "commit", "-am", "Add project files."])
subprocess.check_call(["git", "tag", tag])
generated_sources = rule_runner.request(
GeneratedSources,
[GeneratePythonFromSetuptoolsSCMRequest(EMPTY_SNAPSHOT, tgt)])
assert generated_sources.snapshot.files == ("src/python/_version.py", )
dc = rule_runner.request(DigestContents,
[generated_sources.snapshot.digest])
assert len(dc) == 1
fc = dc[0]
assert fc.path == "src/python/_version.py"
assert fc.content.decode() == f'version = "{expected_version}"'
assert fc.is_executable is False
def _emit_targets(cls, workdir):
with safe_open(os.path.join(workdir, 'BUILD'), 'w') as f:
f.write(
dedent("""
remote_sources(name='proto-7',
dest=java_protobuf_library,
sources_target=':external-source',
args=dict(
platform='java7',
),
)
remote_sources(name='proto-8',
dest=java_protobuf_library,
sources_target=':external-source',
args=dict(
platform='java8',
),
dependencies=[
':proto-sources',
],
)
remote_sources(name='proto-sources',
dest=resources,
sources_target=':external-source',
)
unpacked_jars(name='external-source',
libraries=[':external-source-jars'],
include_patterns=[
'com/squareup/testing/**/*.proto',
],
)
jar_library(name='external-source-jars',
jars=[
jar(org='com.squareup.testing.protolib', name='protolib-external-test', rev='0.0.2'),
],
)
"""))
return [
'{}:proto-{}'.format(os.path.relpath(workdir, get_buildroot()),
num) for num in (7, 8)
]
def test_parse_failed_targets_nominal(self):
registry = RegistryOfTests({JUnitTest('org.pantsbuild.Failure'): 'Bob',
JUnitTest('org.pantsbuild.Error'): 'Jane',
JUnitTest('org.pantsbuild.AnotherError'): 'Bob',
JUnitTest('org.pantsbuild.subpackage.AnotherFailure'): 'Mary'})
with temporary_dir() as junit_xml_dir:
with open(os.path.join(junit_xml_dir, 'TEST-a.xml'), 'w') as fp:
fp.write("""
<testsuite failures="1" errors="1">
<testcase classname="org.pantsbuild.Green" name="testOK"/>
<testcase classname="org.pantsbuild.Failure" name="testFailure">
<failure/>
</testcase>
<testcase classname="org.pantsbuild.Error" name="testError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'TEST-b.xml'), 'w') as fp:
fp.write("""
<testsuite failures="0" errors="1">
<testcase classname="org.pantsbuild.AnotherError" name="testAnotherError">
<error/>
</testcase>
</testsuite>
""")
with open(os.path.join(junit_xml_dir, 'random.xml'), 'w') as fp:
fp.write('<invalid></xml>')
with safe_open(os.path.join(junit_xml_dir, 'subdir', 'TEST-c.xml'), 'w') as fp:
fp.write("""
<testsuite failures="1" errors="0">
<testcase classname="org.pantsbuild.subpackage.AnotherFailure" name="testAnotherFailue">
<failure/>
</testcase>
</testsuite>
""")
failed_targets = parse_failed_targets(registry, junit_xml_dir, self._raise_handler)
self.assertEqual({'Bob': {JUnitTest('org.pantsbuild.Failure', 'testFailure'),
JUnitTest('org.pantsbuild.AnotherError', 'testAnotherError')},
'Jane': {JUnitTest('org.pantsbuild.Error', 'testError')},
'Mary': {JUnitTest('org.pantsbuild.subpackage.AnotherFailure',
'testAnotherFailue')}},
failed_targets)
def run_pants_with_workdir(self,
command,
workdir,
config=None,
stdin_data=None,
extra_env=None,
**kwargs):
config = config.copy() if config else {}
# We add workdir to the DEFAULT section, and also ensure that it's emitted first.
default_section = config.pop('DEFAULT', {})
default_section['pants_workdir'] = '%s' % workdir
ini = ''
for section, section_config in [('DEFAULT', default_section)
] + config.items():
ini += '\n[%s]\n' % section
for key, val in section_config.items():
ini += '%s: %s\n' % (key, val)
ini_file_name = os.path.join(workdir, 'pants.ini')
with safe_open(ini_file_name, mode='w') as fp:
fp.write(ini)
env = os.environ.copy()
env.update(extra_env or {})
# TODO: We can replace the env var with a '--config-override={0}'.format(ini_file_name) arg,
# once we're rid of the special-case handling of this env var in pants_exe.
env['PANTS_CONFIG_OVERRIDE'] = ini_file_name
pants_script = os.path.join(get_buildroot(), self.PANTS_SCRIPT_NAME)
pants_command = [
pants_script, '--no-lock', '--kill-nailguns', '--no-pantsrc',
'--print-exception-stacktrace'
] + command
proc = subprocess.Popen(pants_command,
env=env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs)
(stdout_data, stderr_data) = proc.communicate(stdin_data)
return PantsResult(pants_command, proc.returncode, stdout_data,
stderr_data)
def unpacked_aar_library(self,
location,
manifest=True,
classes_jar=True,
resources=True,
filenames=None):
"""Mock the contents of an aar file, with optional components and additional files."""
if manifest:
manifest_file = os.path.join(location, 'AndroidManifest.xml')
touch(manifest_file)
with safe_open(manifest_file, 'w') as fp:
fp.write(self.android_manifest())
fp.close()
if classes_jar:
self.create_jarfile(location, filenames=filenames)
if resources:
safe_mkdir(os.path.join(location, 'res'))
return location
def _emit_targets(cls, workdir):
prep_command_path = os.path.join(
workdir, 'src/java/org/pantsbuild/prepcommand')
with safe_open(os.path.join(prep_command_path, 'BUILD'), 'w') as fp:
for name, touch_target in cls._SENTINELS.items():
fp.write(
dedent("""
prep_command(
name='{name}',
goals=['{goal}'],
prep_executable='touch',
prep_args=['{tmpdir}/{touch_target}'],
)
""".format(name=name,
goal=name,
tmpdir=workdir,
touch_target=touch_target)))
return [f'{prep_command_path}:{name}' for name in cls._SENTINELS]
def distribution(files=None, executables=None, java_home=None, dist_dir=None):
# NB attempt to include the java version in the tmp dir name for better test failure messages.
executables_as_list = ensure_list(executables or (), expected_type=EXE)
if executables_as_list:
dist_prefix = "jvm_{}_".format(executables_as_list[0]._version)
else:
dist_prefix = "jvm_na_"
with temporary_dir(root_dir=dist_dir, prefix=dist_prefix) as dist_root:
for f in ensure_str_list(files or ()):
touch(os.path.join(dist_root, f))
for executable in executables_as_list:
path = os.path.join(dist_root, executable.relpath)
with safe_open(path, "w") as fp:
java_home = os.path.join(dist_root,
java_home) if java_home else dist_root
fp.write(executable.contents(java_home))
chmod_plus_x(path)
yield dist_root
def generate_ivysettings(self, ivy, publishedjars, publish_local=None):
template_relpath = os.path.join(_TEMPLATES_RELPATH, "ivysettings.xml.mustache")
template_text = pkgutil.get_data(__name__, template_relpath).decode()
published = [TemplateData(org=jar.org, name=jar.name) for jar in publishedjars]
generator = Generator(
template_text,
ivysettings=self.fetch_ivysettings(ivy),
dir=self.workdir,
cachedir=self.cachedir,
published=published,
publish_local=publish_local,
)
with safe_open(os.path.join(self.workdir, "ivysettings.xml"), "w") as wrapper:
generator.write(wrapper)
return wrapper.name
def create_plugin(distribution_repo_dir,
plugin,
version=None,
packager_cls=None):
with safe_open(os.path.join(distribution_repo_dir, plugin, 'setup.py'),
'w') as fp:
fp.write(
dedent(f"""
from setuptools import setup
setup(name="{plugin}", version="{version or '0.0.0'}")
"""))
packager_cls = packager_cls or Packager
packager = packager_cls(source_dir=os.path.join(
distribution_repo_dir, plugin),
install_dir=distribution_repo_dir)
packager.run()
def _get_remote_import_paths(self, pkg, gopath=None):
"""Returns the remote import paths declared by the given Go `pkg`."""
out = self.go_dist.create_go_cmd('list',
args=['-json', pkg],
gopath=gopath).check_output()
try:
data = json.loads(out)
imports = data.get('Imports', [])
imports.extend(data.get('TestImports', []))
return [imp for imp in imports if imp not in self.go_stdlib]
except ValueError as e:
save_file = os.path.join(gopath, '.errors', pkg, 'list.json')
with safe_open(save_file, 'w') as fp:
fp.write(out)
self.context.log.error(
'Problem determining imports for {}, saved json response to {}'
.format(pkg, save_file))
raise TaskError(e)
def test_no_address_no_family(self):
spec = SingleAddress('a/c', 'c')
# Does not exist.
self.assertEqual(0, len(self.resolve(spec)))
# Exists on disk, but not yet in memory.
directory = 'a/c'
build_file = os.path.join(self.build_root, directory, 'c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
self.assertEqual(0, len(self.resolve(spec)))
# Success.
self.scheduler.invalidate_files([directory])
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual(Struct(name='c', type_alias='struct'),
resolved[0].struct)
def _resolve_exact_plugin_locations(self):
hasher = hashlib.sha1()
for req in sorted(self._plugin_requirements):
hasher.update(req)
resolve_hash = hasher.hexdigest()
resolved_plugins_list = os.path.join(
self.plugin_cache_dir, 'plugins-{}.txt'.format(resolve_hash))
if not os.path.exists(resolved_plugins_list):
tmp_plugins_list = resolved_plugins_list + '~'
with safe_open(tmp_plugins_list, 'w') as fp:
for plugin in self._resolve_plugins():
fp.write(plugin.location)
fp.write('\n')
os.rename(tmp_plugins_list, resolved_plugins_list)
with open(resolved_plugins_list) as fp:
for plugin_location in fp:
yield plugin_location.strip()
def __init__(self, *args, **kwargs):
super(ConsoleTask, self).__init__(*args, **kwargs)
separator_option = "console_%s_separator" % self.__class__.__name__
self._console_separator = getattr(
self.context.options, separator_option).decode('string-escape')
if self.context.options.console_outstream:
try:
self._outstream = safe_open(
os.path.abspath(self.context.options.console_outstream),
'w')
except IOError as e:
raise TaskError(
'Error opening stream {out_file} due to'
' {error_str}'.format(
out_file=self.context.options.console_outstream,
error_str=e))
else:
self._outstream = self.context.console_outstream
def test_no_address_no_family(self) -> None:
spec = SingleAddress("a/c", "c")
# Does not exist.
with self.assertRaises(Exception):
self.resolve(spec)
build_file = os.path.join(self.build_root, "a/c", "c.BUILD.json")
with safe_open(build_file, "w") as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
self.scheduler.invalidate_files(["a/c"])
# Success.
resolved = self.resolve(spec)
self.assertEqual(1, len(resolved))
self.assertEqual(
[Struct(address=Address.parse("a/c"), type_alias="struct")],
resolved)
def execute(self):
if not self.url:
raise TaskError('Unable to proceed publishing to confluence. Please set the url option.')
deprecated_conditional(
lambda: True,
'1.6.0.dev0',
'pants.backend.docgen.tasks.confluence_publish.py',
'Use contrib.confluence.tasks.confluence_publish.py instead'
)
pages = []
targets = self.context.targets()
for target in targets:
if isinstance(target, Page):
for wiki_artifact in target.payload.provides:
pages.append((target, wiki_artifact))
urls = list()
genmap = self.context.products.get('wiki_html')
for page, wiki_artifact in pages:
html_info = genmap.get((wiki_artifact, page))
if len(html_info) > 1:
raise TaskError('Unexpected resources for {}: {}'.format(page, html_info))
basedir, htmls = html_info.items()[0]
if len(htmls) != 1:
raise TaskError('Unexpected resources for {}: {}'.format(page, htmls))
with safe_open(os.path.join(basedir, htmls[0])) as contents:
url = self.publish_page(
page.address,
wiki_artifact.config['space'],
wiki_artifact.config['title'],
contents.read(),
# Default to none if not present in the hash.
parent=wiki_artifact.config.get('parent')
)
if url:
urls.append(url)
self.context.log.info('Published {} to {}'.format(page, url))
if self.open and urls:
try:
desktop.ui_open(*urls)
except desktop.OpenError as e:
raise TaskError(e)