def configure_profiles(self, scala_compiler_profile):
checkstyle_enabled = len(Phase.goals_of_type(Checkstyle)) > 0
self.checkstyle_classpath = (binary_util.profile_classpath('checkstyle') if checkstyle_enabled
else [])
self.scala_compiler_classpath = []
if self.has_scala:
self.scala_compiler_classpath.extend(binary_util.profile_classpath(scala_compiler_profile))
def configure_profiles(self, scala_compiler_profile):
checkstyle_enabled = len(Phase.goals_of_type(Checkstyle)) > 0
self.checkstyle_classpath = \
(binary_util.profile_classpath('checkstyle', workunit_factory=self.workunit_factory)
if checkstyle_enabled else [])
self.scala_compiler_classpath = []
if self.has_scala:
self.scala_compiler_classpath.extend(
binary_util.profile_classpath(
scala_compiler_profile,
workunit_factory=self.workunit_factory))
def execute(self, targets):
# The repl session may last a while, allow concurrent pants activity during this pants idle
# period.
self.context.lock.release()
self.save_stty_options()
def repl_workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(name=name, labels=[WorkUnit.REPL] + labels, cmd=cmd)
kwargs = {
'jvmargs': self.jvm_args,
'classpath': self.classpath(profile_classpath(self.profile), confs=self.confs),
'main': self.main,
'args': self.args
}
# Capture the cmd_line.
cmd = runjava_indivisible(dryrun=True, **kwargs)
with self.context.new_workunit(name='repl', labels=[WorkUnit.REPL, WorkUnit.JVM], cmd=cmd):
# Now actually run the REPL. We don't let runjava_indivisible create a workunit because we
# want the REPL's output to go straight to stdout and not get buffered by the report.
print('') # Start REPL output on a new line.
try:
runjava_indivisible(dryrun=False, **kwargs)
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
self.restore_ssty_options()
def __init__(self, context):
Task.__init__(self, context)
self.profile = context.config.get('benchmark-run', 'profile',
default="benchmark-caliper-0.5")
self.confs = context.config.getlist('benchmark-run', 'confs')
self.java_args = context.config.getlist('benchmark-run', 'args',
default=['-Xmx1g', '-XX:MaxPermSize=256m'])
self.agent_profile = context.config.get('benchmark-run', 'agent_profile',
default="benchmark-java-allocation-instrumenter-2.1")
# TODO(Steve Gury):
# Find all the target classes from the Benchmark target itself
# https://jira.twitter.biz/browse/AWESOME-1938
self.caliper_args = context.options.target_class
if context.options.memory_profiling:
self.caliper_args += ['--measureMemory']
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_jar = os.readlink(profile_classpath(self.agent_profile)[0])
allocation_jar = os.path.join(os.path.dirname(agent_jar), "allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
if context.options.debug:
self.java_args.extend(context.config.getlist('jvm', 'debug_args'))
self.caliper_args += ['--debug']
def __init__(self, context):
Task.__init__(self, context)
self.profile = context.config.get('benchmark-run',
'profile',
default="benchmark-caliper-0.5")
self.confs = context.config.getlist('benchmark-run', 'confs')
self.java_args = context.config.getlist(
'benchmark-run',
'args',
default=['-Xmx1g', '-XX:MaxPermSize=256m'])
self.agent_profile = context.config.get(
'benchmark-run',
'agent_profile',
default="benchmark-java-allocation-instrumenter-2.1")
# TODO(Steve Gury):
# Find all the target classes from the Benchmark target itself
# https://jira.twitter.biz/browse/AWESOME-1938
self.caliper_args = context.options.target_class
if context.options.memory_profiling:
self.caliper_args += ['--measureMemory']
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_jar = os.readlink(profile_classpath(self.agent_profile)[0])
allocation_jar = os.path.join(os.path.dirname(agent_jar),
"allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
if context.options.debug:
self.java_args.extend(context.config.getlist('jvm', 'debug_args'))
self.caliper_args += ['--debug']
def profile_classpath(self, profile):
"""Ensures the classpath for the given profile ivy.xml is available and returns it as a list of
paths.
profile: The name of the tool profile classpath to ensure.
"""
# binary_util.profile_classpath wants to pass the workunit_factory into the runner,
# so we give it a wrapper method that accepts that argument.
def java_runner(main,
classpath=None,
opts=None,
args=None,
jvmargs=None,
workunit_factory=None,
workunit_name=None):
assert workunit_factory is None
return self.runjava_indivisible(main,
classpath=classpath,
opts=opts,
args=args,
jvmargs=jvmargs,
workunit_name=workunit_name)
return binary_util.profile_classpath(profile,
java_runner=java_runner,
config=self.context.config)
def _spawn_nailgun_server(self, workunit):
self.context.log.debug("No ng server found, spawning...")
with _safe_open(self._ng_out, "w"):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(workunit)
# NOTE: Don't use self.context.log or self.context.new_workunit here.
# They use threadlocal state, which interacts poorly with fork().
os.setsid()
in_fd = open("/dev/null", "w")
out_fd = safe_open(self._ng_out, "w")
err_fd = safe_open(self._ng_err, "w")
args = ["java"]
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(self._identifier_arg)
ng_classpath = os.pathsep.join(binary_util.profile_classpath(self._nailgun_profile))
args.extend(["-cp", ng_classpath, "com.martiansoftware.nailgun.NGServer", ":0"])
s = " ".join(args)
with binary_util.safe_classpath():
subprocess.Popen(args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot())
# Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
# blocks because we might, e.g., clean up tempfiles that the parent still needs.
os._exit(0)
def execute(self, targets):
exit_code = runjava_indivisible(
jvmargs=self.java_args,
classpath=self.classpath(profile_classpath(self.profile), confs=self.confs),
main='com.google.caliper.Runner',
opts=self.caliper_args
)
if exit_code != 0:
raise TaskError()
def profile_classpath(self, profile):
"""Ensures the classpath for the given profile ivy.xml is available and returns it as a list of
paths.
profile: The name of the tool profile classpath to ensure.
"""
return binary_util.profile_classpath(profile,
java_runner=self.runjava_indivisible,
config=self.context.config)
def execute(self, targets):
exit_code = runjava_indivisible(jvmargs=self.java_args,
classpath=self.classpath(
profile_classpath(self.profile),
confs=self.confs),
main='com.google.caliper.Runner',
opts=self.caliper_args,
workunit_name='caliper')
if exit_code != 0:
raise TaskError()
def run_tests(tests):
opts = ['--color'] if self.color else []
opts.append('--specs=%s' % ','.join(tests))
result = runjava_indivisible(
jvmargs=self.java_args,
classpath=self.classpath(profile_classpath(self.profile), confs=self.confs),
main='com.twitter.common.testing.ExplicitSpecsRunnerMain',
opts=opts
)
if result != 0:
raise TaskError()
def profile_classpath(self, profile):
"""Ensures the classpath for the given profile ivy.xml is available and returns it as a list of
paths.
profile: The name of the tool profile classpath to ensure.
"""
# binary_util.profile_classpath wants to pass the workunit_factory into the runner,
# so we give it a wrapper method that accepts that argument.
def java_runner(main, classpath=None, opts=None, args=None, jvmargs=None,
workunit_factory=None, workunit_name=None):
assert workunit_factory is None
return self.runjava_indivisible(main, classpath=classpath, opts=opts, args=args,
jvmargs=jvmargs, workunit_name=workunit_name)
return binary_util.profile_classpath(profile,
java_runner=java_runner,
config=self.context.config)
def _spawn_nailgun_server(self, workunit):
log.info('No ng server found, spawning...')
with _safe_open(self._ng_out, 'w'):
pass # truncate
if os.path.exists(self._pidfile):
os.remove(
self._pidfile) # So we know when the child has written it.
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(workunit)
os.setsid()
in_fd = open('/dev/null', 'w')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
args = ['java']
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(NailgunTask.create_pidfile_arg(self._pidfile))
ng_classpath = os.pathsep.join(
binary_util.profile_classpath(
self._nailgun_profile,
workunit_factory=self.context.new_workunit))
args.extend([
'-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0'
])
log.debug('Executing: %s' % ' '.join(args))
with binary_util.safe_classpath(logger=log.warn):
process = subprocess.Popen(args,
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True,
cwd=get_buildroot())
with _safe_open(self._pidfile, 'w') as pidfile:
pidfile.write('%d' % process.pid)
log.debug('Spawned ng server @ %d' % process.pid)
# Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
# blocks because we might, e.g., clean up tempfiles that the parent still needs.
os._exit(0)
def run_tests(tests):
def workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd)
opts = ['--color'] if self.color else []
opts.append('--specs=%s' % ','.join(tests))
result = runjava_indivisible(
jvmargs=self.java_args,
classpath=self.classpath(profile_classpath(self.profile), confs=self.confs),
main='com.twitter.common.testing.ExplicitSpecsRunnerMain',
opts=opts,
workunit_factory=workunit_factory,
workunit_name='specs'
)
if result != 0:
raise TaskError()
def execute(self, targets):
# The repl session may last a while, allow concurrent pants activity during this pants idle
# period.
self.context.lock.release()
self.save_stty_options()
try:
runjava_indivisible(
jvmargs=self.jvm_args,
classpath=self.classpath(profile_classpath(self.profile), confs=self.confs),
main=self.main,
args=self.args
)
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
self.restore_ssty_options()
def _spawn_nailgun_server(self, workunit):
log.info('No ng server found, spawning...')
with _safe_open(self._ng_out, 'w'):
pass # truncate
if os.path.exists(self._pidfile):
os.remove(self._pidfile) # So we know when the child has written it.
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(workunit)
os.setsid()
in_fd = open('/dev/null', 'w')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
args = ['java']
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(NailgunTask.create_pidfile_arg(self._pidfile))
ng_classpath = os.pathsep.join(binary_util.profile_classpath(self._nailgun_profile,
workunit_factory=self.context.new_workunit))
args.extend(['-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0'])
log.debug('Executing: %s' % ' '.join(args))
with binary_util.safe_classpath(logger=log.warn):
process = subprocess.Popen(
args,
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True,
cwd=get_buildroot()
)
with _safe_open(self._pidfile, 'w') as pidfile:
pidfile.write('%d' % process.pid)
log.debug('Spawned ng server @ %d' % process.pid)
# Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
# blocks because we might, e.g., clean up tempfiles that the parent still needs.
os._exit(0)
def execute(self, targets):
# The repl session may last a while, allow concurrent pants activity during this pants idle
# period.
self.context.lock.release()
self.save_stty_options()
try:
runjava_indivisible(jvmargs=self.jvm_args,
classpath=self.classpath(profile_classpath(
self.profile),
confs=self.confs),
main=self.main,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='repl')
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
self.restore_ssty_options()
def run_tests(tests):
def workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(name=name,
labels=[WorkUnit.TEST] +
labels,
cmd=cmd)
opts = ['--color'] if self.color else []
opts.append('--specs=%s' % ','.join(tests))
result = runjava_indivisible(
jvmargs=self.java_args,
classpath=self.classpath(profile_classpath(self.profile),
confs=self.confs),
main='com.twitter.common.testing.ExplicitSpecsRunnerMain',
opts=opts,
workunit_factory=workunit_factory,
workunit_name='specs')
if result != 0:
raise TaskError()
def _spawn_nailgun_server(self, workunit):
self.context.log.debug('No ng server found, spawning...')
with _safe_open(self._ng_out, 'w'):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(workunit)
# NOTE: Don't use self.context.log or self.context.new_workunit here.
# They use threadlocal state, which interacts poorly with fork().
os.setsid()
in_fd = open('/dev/null', 'w')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
args = ['java']
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(self._identifier_arg)
ng_classpath = os.pathsep.join(
binary_util.profile_classpath(self._nailgun_profile))
args.extend([
'-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0'
])
s = ' '.join(args)
with binary_util.safe_classpath():
subprocess.Popen(args,
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True,
cwd=get_buildroot())
# Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally
# blocks because we might, e.g., clean up tempfiles that the parent still needs.
os._exit(0)
def execute(self, targets):
if not self.context.options.junit_run_skip:
tests = list(self.normalize_test_classes() if self.
test_classes else self.calculate_tests(targets))
if tests:
junit_classpath = self.classpath(
binary_util.profile_classpath(
self.junit_profile,
workunit_factory=self.context.new_workunit),
confs=self.confs,
exclusives_classpath=self.get_base_classpath_for_target(
targets[0]))
def run_tests(classpath, main, jvmargs=None):
def test_workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(
name=name,
labels=[WorkUnit.TEST] + labels,
cmd=cmd)
# TODO(John Sirois): Integrated batching with the test runner. As things stand we get
# results summaries for example for each batch but no overall summary.
# http://jira.local.twitter.com/browse/AWESOME-1114
result = 0
for batch in self._partition(tests):
with binary_util.safe_args(batch) as batch_tests:
result += abs(
binary_util.runjava_indivisible(
jvmargs=(jvmargs or []) + self.java_args,
classpath=classpath,
main=main,
opts=self.opts,
args=batch_tests,
workunit_factory=test_workunit_factory,
workunit_name='run'))
if result != 0 and self.fail_fast:
break
if result != 0:
raise TaskError()
if self.coverage:
emma_classpath = binary_util.profile_classpath(
self.emma_profile)
def instrument_code():
safe_mkdir(self.coverage_instrument_dir, clean=True)
with binary_util.safe_args(
self.get_coverage_patterns(
targets)) as patterns:
opts = [
'instr', '-out', self.coverage_metadata_file,
'-d', self.coverage_instrument_dir, '-cp',
os.pathsep.join(junit_classpath), '-exit'
]
for pattern in patterns:
opts.extend(['-filter', pattern])
result = binary_util.runjava_indivisible(
classpath=emma_classpath,
main='emma',
opts=opts,
workunit_name='emma')
if result != 0:
raise TaskError(
'Emma instrumentation failed with: %d' %
result)
def generate_reports():
opts = [
'report', '-in', self.coverage_metadata_file,
'-in', self.coverage_file, '-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self.test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
opts.extend(['-sp', source_base])
sorting = [
'-Dreport.sort', '+name,+class,+method,+block'
]
if self.coverage_report_console:
opts.extend([
'-r', 'txt',
'-Dreport.txt.out.file=%s' %
self.coverage_console_file
] + sorting)
if self.coverage_report_xml:
opts.extend([
'-r', 'xml',
'-Dreport.xml.out.file=%s' %
self.coverage_xml_file
])
if self.coverage_report_html:
opts.extend([
'-r', 'html',
'-Dreport.html.out.file=%s' %
self.coverage_html_file,
'-Dreport.out.encoding=UTF-8'
] + sorting)
result = binary_util.runjava_indivisible(
classpath=emma_classpath,
main='emma',
opts=opts,
workunit_name='emma')
if result != 0:
raise TaskError(
'Failed to emma generate code coverage reports: %d'
% result)
if self.coverage_report_console:
with safe_open(self.coverage_console_file
) as console_report:
sys.stdout.write(console_report.read())
if self.coverage_report_html_open:
binary_util.ui_open(self.coverage_html_file)
instrument_code()
try:
# Coverage runs over instrumented classes require the instrumented classes come 1st in
# the classpath followed by the normal classpath. The instrumentation also adds a
# dependency on emma libs that must be satisfied on the classpath.
run_tests(
[self.coverage_instrument_dir] + junit_classpath +
emma_classpath,
'com.twitter.common.testing.runner.JUnitConsoleRunner',
jvmargs=[
'-Demma.coverage.out.file=%s' %
self.coverage_file
])
finally:
generate_reports()
else:
self.context.lock.release()
run_tests(
junit_classpath,
'com.twitter.common.testing.runner.JUnitConsoleRunner')
def classpath_for_profile(profile): return profile_classpath(profile, java_runner=self._java_runner, config=self._context.config)
def _classpth(self, target): compiler_config = INFO_FOR_COMPILER[target.compiler]['config'] return profile_classpath(compiler_config)
def _classpth(self, target):
compiler_config = INFO_FOR_COMPILER[target.compiler]['config']
return profile_classpath(compiler_config)
def classpath_for_profile(profile):
return profile_classpath(profile,
java_runner=self._java_runner,
config=self._context.config)
def execute(self, targets):
if not self.context.options.junit_run_skip:
tests = list(self.normalize_test_classes() if self.test_classes
else self.calculate_tests(targets))
if tests:
junit_classpath = self.classpath(binary_util.profile_classpath(self.junit_profile,
workunit_factory=self.context.new_workunit), confs=self.confs,
exclusives_classpath=self.get_base_classpath_for_target(targets[0]))
def run_tests(classpath, main, jvmargs=None):
def test_workunit_factory(name, labels=list(), cmd=''):
return self.context.new_workunit(name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd)
# TODO(John Sirois): Integrated batching with the test runner. As things stand we get
# results summaries for example for each batch but no overall summary.
# http://jira.local.twitter.com/browse/AWESOME-1114
result = 0
for batch in self._partition(tests):
with binary_util.safe_args(batch) as batch_tests:
result += abs(binary_util.runjava_indivisible(
jvmargs=(jvmargs or []) + self.java_args,
classpath=classpath,
main=main,
opts=self.opts,
args=batch_tests,
workunit_factory=test_workunit_factory,
workunit_name='run'
))
if result != 0 and self.fail_fast:
break
if result != 0:
raise TaskError()
if self.coverage:
emma_classpath = binary_util.profile_classpath(self.emma_profile)
def instrument_code():
safe_mkdir(self.coverage_instrument_dir, clean=True)
with binary_util.safe_args(self.get_coverage_patterns(targets)) as patterns:
opts = [
'instr',
'-out', self.coverage_metadata_file,
'-d', self.coverage_instrument_dir,
'-cp', os.pathsep.join(junit_classpath),
'-exit'
]
for pattern in patterns:
opts.extend(['-filter', pattern])
result = binary_util.runjava_indivisible(classpath=emma_classpath, main='emma',
opts=opts, workunit_name='emma')
if result != 0:
raise TaskError('Emma instrumentation failed with: %d' % result)
def generate_reports():
opts = [
'report',
'-in', self.coverage_metadata_file,
'-in', self.coverage_file,
'-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self.test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
opts.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
if self.coverage_report_console:
opts.extend(['-r', 'txt',
'-Dreport.txt.out.file=%s' % self.coverage_console_file] + sorting)
if self.coverage_report_xml:
opts.extend(['-r', 'xml','-Dreport.xml.out.file=%s' % self.coverage_xml_file])
if self.coverage_report_html:
opts.extend(['-r', 'html',
'-Dreport.html.out.file=%s' % self.coverage_html_file,
'-Dreport.out.encoding=UTF-8'] + sorting)
result = binary_util.runjava_indivisible(
classpath=emma_classpath,
main='emma',
opts=opts,
workunit_name='emma'
)
if result != 0:
raise TaskError('Failed to emma generate code coverage reports: %d' % result)
if self.coverage_report_console:
with safe_open(self.coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self.coverage_report_html_open:
binary_util.ui_open(self.coverage_html_file)
instrument_code()
try:
# Coverage runs over instrumented classes require the instrumented classes come 1st in
# the classpath followed by the normal classpath. The instrumentation also adds a
# dependency on emma libs that must be satisfied on the classpath.
run_tests(
[self.coverage_instrument_dir] + junit_classpath + emma_classpath,
'com.twitter.common.testing.runner.JUnitConsoleRunner',
jvmargs=['-Demma.coverage.out.file=%s' % self.coverage_file]
)
finally:
generate_reports()
else:
self.context.lock.release()
run_tests(junit_classpath, 'com.twitter.common.testing.runner.JUnitConsoleRunner')
