def configure_profiles(self, scala_compiler_profile): checkstyle_enabled = len(Phase.goals_of_type(Checkstyle)) > 0 self.checkstyle_classpath = (binary_util.profile_classpath('checkstyle') if checkstyle_enabled else []) self.scala_compiler_classpath = [] if self.has_scala: self.scala_compiler_classpath.extend(binary_util.profile_classpath(scala_compiler_profile))
def configure_profiles(self, scala_compiler_profile): checkstyle_enabled = len(Phase.goals_of_type(Checkstyle)) > 0 self.checkstyle_classpath = \ (binary_util.profile_classpath('checkstyle', workunit_factory=self.workunit_factory) if checkstyle_enabled else []) self.scala_compiler_classpath = [] if self.has_scala: self.scala_compiler_classpath.extend( binary_util.profile_classpath( scala_compiler_profile, workunit_factory=self.workunit_factory))
def execute(self, targets): # The repl session may last a while, allow concurrent pants activity during this pants idle # period. self.context.lock.release() self.save_stty_options() def repl_workunit_factory(name, labels=list(), cmd=''): return self.context.new_workunit(name=name, labels=[WorkUnit.REPL] + labels, cmd=cmd) kwargs = { 'jvmargs': self.jvm_args, 'classpath': self.classpath(profile_classpath(self.profile), confs=self.confs), 'main': self.main, 'args': self.args } # Capture the cmd_line. cmd = runjava_indivisible(dryrun=True, **kwargs) with self.context.new_workunit(name='repl', labels=[WorkUnit.REPL, WorkUnit.JVM], cmd=cmd): # Now actually run the REPL. We don't let runjava_indivisible create a workunit because we # want the REPL's output to go straight to stdout and not get buffered by the report. print('') # Start REPL output on a new line. try: runjava_indivisible(dryrun=False, **kwargs) except KeyboardInterrupt: # TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an # explicit catch of KeyboardInterrupt is required. pass self.restore_ssty_options()
def __init__(self, context): Task.__init__(self, context) self.profile = context.config.get('benchmark-run', 'profile', default="benchmark-caliper-0.5") self.confs = context.config.getlist('benchmark-run', 'confs') self.java_args = context.config.getlist('benchmark-run', 'args', default=['-Xmx1g', '-XX:MaxPermSize=256m']) self.agent_profile = context.config.get('benchmark-run', 'agent_profile', default="benchmark-java-allocation-instrumenter-2.1") # TODO(Steve Gury): # Find all the target classes from the Benchmark target itself # https://jira.twitter.biz/browse/AWESOME-1938 self.caliper_args = context.options.target_class if context.options.memory_profiling: self.caliper_args += ['--measureMemory'] # For rewriting JDK classes to work, the JAR file has to be listed specifically in # the JAR manifest as something that goes in the bootclasspath. # The MANIFEST list a jar 'allocation.jar' this is why we have to rename it agent_jar = os.readlink(profile_classpath(self.agent_profile)[0]) allocation_jar = os.path.join(os.path.dirname(agent_jar), "allocation.jar") # TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient # to version upgrade shutil.copyfile(agent_jar, allocation_jar) os.environ['ALLOCATION_JAR'] = str(allocation_jar) if context.options.debug: self.java_args.extend(context.config.getlist('jvm', 'debug_args')) self.caliper_args += ['--debug']
def __init__(self, context): Task.__init__(self, context) self.profile = context.config.get('benchmark-run', 'profile', default="benchmark-caliper-0.5") self.confs = context.config.getlist('benchmark-run', 'confs') self.java_args = context.config.getlist( 'benchmark-run', 'args', default=['-Xmx1g', '-XX:MaxPermSize=256m']) self.agent_profile = context.config.get( 'benchmark-run', 'agent_profile', default="benchmark-java-allocation-instrumenter-2.1") # TODO(Steve Gury): # Find all the target classes from the Benchmark target itself # https://jira.twitter.biz/browse/AWESOME-1938 self.caliper_args = context.options.target_class if context.options.memory_profiling: self.caliper_args += ['--measureMemory'] # For rewriting JDK classes to work, the JAR file has to be listed specifically in # the JAR manifest as something that goes in the bootclasspath. # The MANIFEST list a jar 'allocation.jar' this is why we have to rename it agent_jar = os.readlink(profile_classpath(self.agent_profile)[0]) allocation_jar = os.path.join(os.path.dirname(agent_jar), "allocation.jar") # TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient # to version upgrade shutil.copyfile(agent_jar, allocation_jar) os.environ['ALLOCATION_JAR'] = str(allocation_jar) if context.options.debug: self.java_args.extend(context.config.getlist('jvm', 'debug_args')) self.caliper_args += ['--debug']
def profile_classpath(self, profile): """Ensures the classpath for the given profile ivy.xml is available and returns it as a list of paths. profile: The name of the tool profile classpath to ensure. """ # binary_util.profile_classpath wants to pass the workunit_factory into the runner, # so we give it a wrapper method that accepts that argument. def java_runner(main, classpath=None, opts=None, args=None, jvmargs=None, workunit_factory=None, workunit_name=None): assert workunit_factory is None return self.runjava_indivisible(main, classpath=classpath, opts=opts, args=args, jvmargs=jvmargs, workunit_name=workunit_name) return binary_util.profile_classpath(profile, java_runner=java_runner, config=self.context.config)
def _spawn_nailgun_server(self, workunit): self.context.log.debug("No ng server found, spawning...") with _safe_open(self._ng_out, "w"): pass # truncate pid = os.fork() if pid != 0: # In the parent tine - block on ng being up for connections return self._await_nailgun_server(workunit) # NOTE: Don't use self.context.log or self.context.new_workunit here. # They use threadlocal state, which interacts poorly with fork(). os.setsid() in_fd = open("/dev/null", "w") out_fd = safe_open(self._ng_out, "w") err_fd = safe_open(self._ng_err, "w") args = ["java"] if self._ng_server_args: args.extend(self._ng_server_args) args.append(NailgunTask.PANTS_NG_ARG) args.append(self._identifier_arg) ng_classpath = os.pathsep.join(binary_util.profile_classpath(self._nailgun_profile)) args.extend(["-cp", ng_classpath, "com.martiansoftware.nailgun.NGServer", ":0"]) s = " ".join(args) with binary_util.safe_classpath(): subprocess.Popen(args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot()) # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally # blocks because we might, e.g., clean up tempfiles that the parent still needs. os._exit(0)
def execute(self, targets): exit_code = runjava_indivisible( jvmargs=self.java_args, classpath=self.classpath(profile_classpath(self.profile), confs=self.confs), main='com.google.caliper.Runner', opts=self.caliper_args ) if exit_code != 0: raise TaskError()
def profile_classpath(self, profile): """Ensures the classpath for the given profile ivy.xml is available and returns it as a list of paths. profile: The name of the tool profile classpath to ensure. """ return binary_util.profile_classpath(profile, java_runner=self.runjava_indivisible, config=self.context.config)
def execute(self, targets): exit_code = runjava_indivisible(jvmargs=self.java_args, classpath=self.classpath( profile_classpath(self.profile), confs=self.confs), main='com.google.caliper.Runner', opts=self.caliper_args, workunit_name='caliper') if exit_code != 0: raise TaskError()
def run_tests(tests): opts = ['--color'] if self.color else [] opts.append('--specs=%s' % ','.join(tests)) result = runjava_indivisible( jvmargs=self.java_args, classpath=self.classpath(profile_classpath(self.profile), confs=self.confs), main='com.twitter.common.testing.ExplicitSpecsRunnerMain', opts=opts ) if result != 0: raise TaskError()
def _spawn_nailgun_server(self, workunit): log.info('No ng server found, spawning...') with _safe_open(self._ng_out, 'w'): pass # truncate if os.path.exists(self._pidfile): os.remove( self._pidfile) # So we know when the child has written it. pid = os.fork() if pid != 0: # In the parent tine - block on ng being up for connections return self._await_nailgun_server(workunit) os.setsid() in_fd = open('/dev/null', 'w') out_fd = safe_open(self._ng_out, 'w') err_fd = safe_open(self._ng_err, 'w') args = ['java'] if self._ng_server_args: args.extend(self._ng_server_args) args.append(NailgunTask.PANTS_NG_ARG) args.append(NailgunTask.create_pidfile_arg(self._pidfile)) ng_classpath = os.pathsep.join( binary_util.profile_classpath( self._nailgun_profile, workunit_factory=self.context.new_workunit)) args.extend([ '-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0' ]) log.debug('Executing: %s' % ' '.join(args)) with binary_util.safe_classpath(logger=log.warn): process = subprocess.Popen(args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot()) with _safe_open(self._pidfile, 'w') as pidfile: pidfile.write('%d' % process.pid) log.debug('Spawned ng server @ %d' % process.pid) # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally # blocks because we might, e.g., clean up tempfiles that the parent still needs. os._exit(0)
def run_tests(tests): def workunit_factory(name, labels=list(), cmd=''): return self.context.new_workunit(name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd) opts = ['--color'] if self.color else [] opts.append('--specs=%s' % ','.join(tests)) result = runjava_indivisible( jvmargs=self.java_args, classpath=self.classpath(profile_classpath(self.profile), confs=self.confs), main='com.twitter.common.testing.ExplicitSpecsRunnerMain', opts=opts, workunit_factory=workunit_factory, workunit_name='specs' ) if result != 0: raise TaskError()
def execute(self, targets): # The repl session may last a while, allow concurrent pants activity during this pants idle # period. self.context.lock.release() self.save_stty_options() try: runjava_indivisible( jvmargs=self.jvm_args, classpath=self.classpath(profile_classpath(self.profile), confs=self.confs), main=self.main, args=self.args ) except KeyboardInterrupt: # TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an # explicit catch of KeyboardInterrupt is required. pass self.restore_ssty_options()
def _spawn_nailgun_server(self, workunit): log.info('No ng server found, spawning...') with _safe_open(self._ng_out, 'w'): pass # truncate if os.path.exists(self._pidfile): os.remove(self._pidfile) # So we know when the child has written it. pid = os.fork() if pid != 0: # In the parent tine - block on ng being up for connections return self._await_nailgun_server(workunit) os.setsid() in_fd = open('/dev/null', 'w') out_fd = safe_open(self._ng_out, 'w') err_fd = safe_open(self._ng_err, 'w') args = ['java'] if self._ng_server_args: args.extend(self._ng_server_args) args.append(NailgunTask.PANTS_NG_ARG) args.append(NailgunTask.create_pidfile_arg(self._pidfile)) ng_classpath = os.pathsep.join(binary_util.profile_classpath(self._nailgun_profile, workunit_factory=self.context.new_workunit)) args.extend(['-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0']) log.debug('Executing: %s' % ' '.join(args)) with binary_util.safe_classpath(logger=log.warn): process = subprocess.Popen( args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot() ) with _safe_open(self._pidfile, 'w') as pidfile: pidfile.write('%d' % process.pid) log.debug('Spawned ng server @ %d' % process.pid) # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally # blocks because we might, e.g., clean up tempfiles that the parent still needs. os._exit(0)
def execute(self, targets): # The repl session may last a while, allow concurrent pants activity during this pants idle # period. self.context.lock.release() self.save_stty_options() try: runjava_indivisible(jvmargs=self.jvm_args, classpath=self.classpath(profile_classpath( self.profile), confs=self.confs), main=self.main, args=self.args, workunit_factory=self.context.new_workunit, workunit_name='repl') except KeyboardInterrupt: # TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an # explicit catch of KeyboardInterrupt is required. pass self.restore_ssty_options()
def run_tests(tests): def workunit_factory(name, labels=list(), cmd=''): return self.context.new_workunit(name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd) opts = ['--color'] if self.color else [] opts.append('--specs=%s' % ','.join(tests)) result = runjava_indivisible( jvmargs=self.java_args, classpath=self.classpath(profile_classpath(self.profile), confs=self.confs), main='com.twitter.common.testing.ExplicitSpecsRunnerMain', opts=opts, workunit_factory=workunit_factory, workunit_name='specs') if result != 0: raise TaskError()
def _spawn_nailgun_server(self, workunit): self.context.log.debug('No ng server found, spawning...') with _safe_open(self._ng_out, 'w'): pass # truncate pid = os.fork() if pid != 0: # In the parent tine - block on ng being up for connections return self._await_nailgun_server(workunit) # NOTE: Don't use self.context.log or self.context.new_workunit here. # They use threadlocal state, which interacts poorly with fork(). os.setsid() in_fd = open('/dev/null', 'w') out_fd = safe_open(self._ng_out, 'w') err_fd = safe_open(self._ng_err, 'w') args = ['java'] if self._ng_server_args: args.extend(self._ng_server_args) args.append(NailgunTask.PANTS_NG_ARG) args.append(self._identifier_arg) ng_classpath = os.pathsep.join( binary_util.profile_classpath(self._nailgun_profile)) args.extend([ '-cp', ng_classpath, 'com.martiansoftware.nailgun.NGServer', ':0' ]) s = ' '.join(args) with binary_util.safe_classpath(): subprocess.Popen(args, stdin=in_fd, stdout=out_fd, stderr=err_fd, close_fds=True, cwd=get_buildroot()) # Prevents finally blocks being executed, unlike sys.exit(). We don't want to execute finally # blocks because we might, e.g., clean up tempfiles that the parent still needs. os._exit(0)
def execute(self, targets): if not self.context.options.junit_run_skip: tests = list(self.normalize_test_classes() if self. test_classes else self.calculate_tests(targets)) if tests: junit_classpath = self.classpath( binary_util.profile_classpath( self.junit_profile, workunit_factory=self.context.new_workunit), confs=self.confs, exclusives_classpath=self.get_base_classpath_for_target( targets[0])) def run_tests(classpath, main, jvmargs=None): def test_workunit_factory(name, labels=list(), cmd=''): return self.context.new_workunit( name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd) # TODO(John Sirois): Integrated batching with the test runner. As things stand we get # results summaries for example for each batch but no overall summary. # http://jira.local.twitter.com/browse/AWESOME-1114 result = 0 for batch in self._partition(tests): with binary_util.safe_args(batch) as batch_tests: result += abs( binary_util.runjava_indivisible( jvmargs=(jvmargs or []) + self.java_args, classpath=classpath, main=main, opts=self.opts, args=batch_tests, workunit_factory=test_workunit_factory, workunit_name='run')) if result != 0 and self.fail_fast: break if result != 0: raise TaskError() if self.coverage: emma_classpath = binary_util.profile_classpath( self.emma_profile) def instrument_code(): safe_mkdir(self.coverage_instrument_dir, clean=True) with binary_util.safe_args( self.get_coverage_patterns( targets)) as patterns: opts = [ 'instr', '-out', self.coverage_metadata_file, '-d', self.coverage_instrument_dir, '-cp', os.pathsep.join(junit_classpath), '-exit' ] for pattern in patterns: opts.extend(['-filter', pattern]) result = binary_util.runjava_indivisible( classpath=emma_classpath, main='emma', opts=opts, workunit_name='emma') if result != 0: raise TaskError( 'Emma instrumentation failed with: %d' % result) def generate_reports(): opts = [ 'report', '-in', self.coverage_metadata_file, '-in', self.coverage_file, '-exit' ] source_bases = set() def collect_source_base(target): if self.is_coverage_target(target): source_bases.add(target.target_base) for target in self.test_target_candidates(targets): target.walk(collect_source_base) for source_base in source_bases: opts.extend(['-sp', source_base]) sorting = [ '-Dreport.sort', '+name,+class,+method,+block' ] if self.coverage_report_console: opts.extend([ '-r', 'txt', '-Dreport.txt.out.file=%s' % self.coverage_console_file ] + sorting) if self.coverage_report_xml: opts.extend([ '-r', 'xml', '-Dreport.xml.out.file=%s' % self.coverage_xml_file ]) if self.coverage_report_html: opts.extend([ '-r', 'html', '-Dreport.html.out.file=%s' % self.coverage_html_file, '-Dreport.out.encoding=UTF-8' ] + sorting) result = binary_util.runjava_indivisible( classpath=emma_classpath, main='emma', opts=opts, workunit_name='emma') if result != 0: raise TaskError( 'Failed to emma generate code coverage reports: %d' % result) if self.coverage_report_console: with safe_open(self.coverage_console_file ) as console_report: sys.stdout.write(console_report.read()) if self.coverage_report_html_open: binary_util.ui_open(self.coverage_html_file) instrument_code() try: # Coverage runs over instrumented classes require the instrumented classes come 1st in # the classpath followed by the normal classpath. The instrumentation also adds a # dependency on emma libs that must be satisfied on the classpath. run_tests( [self.coverage_instrument_dir] + junit_classpath + emma_classpath, 'com.twitter.common.testing.runner.JUnitConsoleRunner', jvmargs=[ '-Demma.coverage.out.file=%s' % self.coverage_file ]) finally: generate_reports() else: self.context.lock.release() run_tests( junit_classpath, 'com.twitter.common.testing.runner.JUnitConsoleRunner')
def classpath_for_profile(profile): return profile_classpath(profile, java_runner=self._java_runner, config=self._context.config)
def _classpth(self, target): compiler_config = INFO_FOR_COMPILER[target.compiler]['config'] return profile_classpath(compiler_config)
def execute(self, targets): if not self.context.options.junit_run_skip: tests = list(self.normalize_test_classes() if self.test_classes else self.calculate_tests(targets)) if tests: junit_classpath = self.classpath(binary_util.profile_classpath(self.junit_profile, workunit_factory=self.context.new_workunit), confs=self.confs, exclusives_classpath=self.get_base_classpath_for_target(targets[0])) def run_tests(classpath, main, jvmargs=None): def test_workunit_factory(name, labels=list(), cmd=''): return self.context.new_workunit(name=name, labels=[WorkUnit.TEST] + labels, cmd=cmd) # TODO(John Sirois): Integrated batching with the test runner. As things stand we get # results summaries for example for each batch but no overall summary. # http://jira.local.twitter.com/browse/AWESOME-1114 result = 0 for batch in self._partition(tests): with binary_util.safe_args(batch) as batch_tests: result += abs(binary_util.runjava_indivisible( jvmargs=(jvmargs or []) + self.java_args, classpath=classpath, main=main, opts=self.opts, args=batch_tests, workunit_factory=test_workunit_factory, workunit_name='run' )) if result != 0 and self.fail_fast: break if result != 0: raise TaskError() if self.coverage: emma_classpath = binary_util.profile_classpath(self.emma_profile) def instrument_code(): safe_mkdir(self.coverage_instrument_dir, clean=True) with binary_util.safe_args(self.get_coverage_patterns(targets)) as patterns: opts = [ 'instr', '-out', self.coverage_metadata_file, '-d', self.coverage_instrument_dir, '-cp', os.pathsep.join(junit_classpath), '-exit' ] for pattern in patterns: opts.extend(['-filter', pattern]) result = binary_util.runjava_indivisible(classpath=emma_classpath, main='emma', opts=opts, workunit_name='emma') if result != 0: raise TaskError('Emma instrumentation failed with: %d' % result) def generate_reports(): opts = [ 'report', '-in', self.coverage_metadata_file, '-in', self.coverage_file, '-exit' ] source_bases = set() def collect_source_base(target): if self.is_coverage_target(target): source_bases.add(target.target_base) for target in self.test_target_candidates(targets): target.walk(collect_source_base) for source_base in source_bases: opts.extend(['-sp', source_base]) sorting = ['-Dreport.sort', '+name,+class,+method,+block'] if self.coverage_report_console: opts.extend(['-r', 'txt', '-Dreport.txt.out.file=%s' % self.coverage_console_file] + sorting) if self.coverage_report_xml: opts.extend(['-r', 'xml','-Dreport.xml.out.file=%s' % self.coverage_xml_file]) if self.coverage_report_html: opts.extend(['-r', 'html', '-Dreport.html.out.file=%s' % self.coverage_html_file, '-Dreport.out.encoding=UTF-8'] + sorting) result = binary_util.runjava_indivisible( classpath=emma_classpath, main='emma', opts=opts, workunit_name='emma' ) if result != 0: raise TaskError('Failed to emma generate code coverage reports: %d' % result) if self.coverage_report_console: with safe_open(self.coverage_console_file) as console_report: sys.stdout.write(console_report.read()) if self.coverage_report_html_open: binary_util.ui_open(self.coverage_html_file) instrument_code() try: # Coverage runs over instrumented classes require the instrumented classes come 1st in # the classpath followed by the normal classpath. The instrumentation also adds a # dependency on emma libs that must be satisfied on the classpath. run_tests( [self.coverage_instrument_dir] + junit_classpath + emma_classpath, 'com.twitter.common.testing.runner.JUnitConsoleRunner', jvmargs=['-Demma.coverage.out.file=%s' % self.coverage_file] ) finally: generate_reports() else: self.context.lock.release() run_tests(junit_classpath, 'com.twitter.common.testing.runner.JUnitConsoleRunner')