def compile(self, classpath, sources, fingerprint): safe_mkdir(self._classes_dir) jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile) args = [ '-classpath', ':'.join(classpath), '-d', self._classes_dir, '-pdb', os.path.join(self._classes_dir, '%s.dependencies.pdb' % fingerprint), ] compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile) args.extend([ '-jcpath', ':'.join(compiler_classpath), '-jcmainclass', 'com.twitter.common.tools.Compiler', '-C-Tdependencyfile', '-C%s' % self._dependencies_file, ]) args.extend(self._args) args.extend(sources) log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args))) return self.runjava(_JMAKE_MAIN, classpath=jmake_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint, depfile): jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile) args = [ "-classpath", ":".join(classpath), "-d", self._classes_dir, "-pdb", os.path.join(self._classes_dir, "%s.dependencies.pdb" % fingerprint), ] compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile) args.extend( [ "-jcpath", ":".join(compiler_classpath), "-jcmainclass", "com.twitter.common.tools.Compiler", "-C-Tdependencyfile", "-C%s" % depfile, ] ) args.extend(self._args) args.extend(sources) log.debug("Executing: %s %s" % (_JMAKE_MAIN, " ".join(args))) return self.runjava(_JMAKE_MAIN, classpath=jmake_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile): compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) # To pass options to scalac simply prefix with -S. args = ['-S' + x for x in self._args] def analysis_cache_full_path(analysis_cache_product): # We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}. if len(analysis_cache_product) != 1: raise TaskError( 'There can only be one analysis cache file per output directory' ) analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems( ).next() if len(analysis_cache_files) != 1: raise TaskError( 'There can only be one analysis cache file per output directory' ) return os.path.join(analysis_cache_dir, analysis_cache_files[0]) # Strings of <output dir>:<full path to analysis cache file for the classes in that dir>. analysis_map = \ OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ]) if len(analysis_map) > 0: args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in analysis_map.items()]) ]) zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile) zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath) for (name, jarpath) in zinc_jars.items( ): # The zinc jar names are also the flag names. args.extend(['-%s' % name, jarpath]) args.extend([ '-analysis-cache', analysis_cache, '-log-level', self.context.options.log_level or 'info', '-classpath', ':'.join(zinc_classpath + classpath), '-output-products', depfile, '-d', output_dir ]) if not self._color: args.append('-no-color') args.extend(sources) self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args))) return self.runjava(self._main, classpath=zinc_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile): safe_mkdir(output_dir) compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) compiler_args = [] # TODO(John Sirois): separate compiler profile from runtime profile compiler_args.extend([ # Support for outputting a dependencies file of source -> class '-Xplugin:%s' % self.get_depemitter_plugin(), '-P:depemitter:file:%s' % depfile ]) compiler_args.extend(self._args) # To pass options to scalac simply prefix with -S. args = ['-S' + x for x in compiler_args] def analysis_cache_full_path(analysis_cache_product): # We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}. if len(analysis_cache_product) != 1: raise TaskError('There can only be one analysis cache file per output directory') analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems().next() if len(analysis_cache_files) != 1: raise TaskError('There can only be one analysis cache file per output directory') return os.path.join(analysis_cache_dir, analysis_cache_files[0]) # Strings of <output dir>:<full path to analysis cache file for the classes in that dir>. analysis_map = \ OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ]) if len(analysis_map) > 0: args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in analysis_map.items()]) ]) upstream_classes_dirs = analysis_map.keys() zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile) zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath) for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names. args.extend(['-%s' % name, jarpath]) args.extend([ '-analysis-cache', analysis_cache, '-log-level', self.context.options.log_level or 'info', '-classpath', ':'.join(zinc_classpath + classpath + upstream_classes_dirs), '-d', output_dir ]) if not self._color: args.append('-no-color') args.extend(sources) self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args))) return self.runjava(self._main, classpath=zinc_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, bases, sources): safe_mkdir(self._output_dir) compiler_classpath = (self._compiler_classpath or nailgun_profile_classpath( self, self._compile_profile)) self.ng('ng-cp', *compiler_classpath) # TODO(John Sirois): separate compiler profile from runtime profile args = [ '-classpath', ':'.join(compiler_classpath + classpath), '-sourcepath', ':'.join(bases), '-d', self._output_dir, # TODO(John Sirois): dependencyfile requires the deprecated -make:XXX - transition to ssc '-dependencyfile', self._depfile, '-make:transitivenocp' ] args.extend(self._args) args.extend(sources) self.context.log.debug( 'Executing: %s %s' % (self._main, ' '.join(args))) return self.ng(self._main, *args)
def compile(self, classpath, sources): safe_mkdir(self._classes_dir) compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) # TODO(John Sirois): separate compiler profile from runtime profile args = [ '-classpath', ':'.join(compiler_classpath + classpath), '-d', self._classes_dir, # Support for outputting a dependencies file of source -> class '-Xplugin:%s' % self.get_depemitter_plugin(), '-P:depemitter:file:%s' % self._depfile ] args.extend(self._args) args.extend(sources) self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args))) return self.runjava(self._main, classpath=compiler_classpath, args=args)
def _generate_ivy_report(self): classpath = binary_utils.nailgun_profile_classpath(self, self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args) reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_utils.open(*reports)
def _generate_ivy_report(self): classpath = binary_utils.nailgun_profile_classpath(self, self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict( org=org, name=name, conf=conf ) xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args) reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_utils.open(*reports)
def get_depemitter_plugin(self): depemitter_classpath = nailgun_profile_classpath(self, self._depemitter_profile) depemitter_jar = depemitter_classpath.pop() if depemitter_classpath: raise TaskError('Expected only 1 jar for the depemitter plugin, ' 'found these extra: ' % depemitter_classpath) return depemitter_jar
def get_depemitter_plugin(self): depemitter_classpath = nailgun_profile_classpath( self, self._depemitter_profile) depemitter_jar = depemitter_classpath.pop() if depemitter_classpath: raise TaskError('Expected only 1 jar for the depemitter plugin, ' 'found these extra: ' % depemitter_classpath) return depemitter_jar
def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile): safe_mkdir(output_dir) compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) compiler_args = [] # TODO(John Sirois): separate compiler profile from runtime profile compiler_args.extend([ # Support for outputting a dependencies file of source -> class '-Xplugin:%s' % self.get_depemitter_plugin(), '-P:depemitter:file:%s' % depfile ]) compiler_args.extend(self._args) if self._incremental: # To pass options to scalac simply prefix with -S. args = ['-S' + x for x in compiler_args] if len(upstream_analysis_caches) > 0: args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in upstream_analysis_caches.items()]) ]) upstream_jars = upstream_analysis_caches.keys() zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile) zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath) for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names. args.extend(['-%s' % name, jarpath]) args.extend([ '-analysis-cache', analysis_cache, '-log-level', self.context.options.log_level or 'info', '-classpath', ':'.join(zinc_classpath + classpath + upstream_jars) ]) run_classpath = zinc_classpath else: args = compiler_args + ['-classpath', ':'.join(compiler_classpath + classpath)] run_classpath = compiler_classpath args.extend([ '-d', output_dir ]) args.extend(sources) self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args))) return self.runjava(self._main, classpath=run_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint, depfile): jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile) args = [ '-classpath', ':'.join(classpath), '-d', self._classes_dir, '-pdb', os.path.join(self._classes_dir, '%s.dependencies.pdb' % fingerprint), ] compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile) args.extend([ '-jcpath', ':'.join(compiler_classpath), '-jcmainclass', 'com.twitter.common.tools.Compiler', '-C-Tdependencyfile', '-C%s' % depfile, ]) args.extend(self._args) args.extend(sources) log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args))) return self.runjava(_JMAKE_MAIN, classpath=jmake_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint): safe_mkdir(self._classes_dir) jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile) self.ng('ng-cp', *jmake_classpath) args = [ '-classpath', ':'.join(classpath), '-d', self._classes_dir, '-pdb', os.path.join(self._classes_dir, '%s.dependencies.pdb' % fingerprint), ] compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile) args.extend([ '-jcpath', ':'.join(compiler_classpath), '-jcmainclass', 'com.twitter.common.tools.Compiler', '-C-dependencyfile', '-C%s' % self._dependencies_file ]) args.extend(self._args) args.extend(sources) log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args))) return self.ng(_JMAKE_MAIN, *args)
def checkstyle(self, sources): classpath = nailgun_profile_classpath(self, self._profile) with self.context.state('classpath', []) as cp: classpath.extend(jar for conf, jar in cp if conf in self._confs) args = ['-c', self._configuration_file, '-f', 'plain'] if self._properties: properties_file = os.path.join(self._work_dir, 'checkstyle.properties') with safe_open(properties_file, 'w') as pf: for k, v in self._properties.items(): pf.write('%s=%s\n' % (k, v)) args.extend(['-p', properties_file]) args.extend(sources) log.debug('Executing: %s %s' % (CHECKSTYLE_MAIN, ' '.join(args))) return self.runjava(CHECKSTYLE_MAIN, classpath=classpath, args=args)
def checkstyle(self, sources): classpath = nailgun_profile_classpath(self, self._profile) with self.context.state('classpath', []) as cp: classpath.extend(jar for conf, jar in cp if conf in self._confs) args = [ '-c', self._configuration_file, '-f', 'plain' ] if self._properties: properties_file = os.path.join(self._work_dir, 'checkstyle.properties') with safe_open(properties_file, 'w') as pf: for k, v in self._properties.items(): pf.write('%s=%s\n' % (k, v)) args.extend(['-p', properties_file]) args.extend(sources) log.debug('Executing: %s %s' % (CHECKSTYLE_MAIN, ' '.join(args))) return self.runjava(CHECKSTYLE_MAIN, classpath=classpath, args=args)
def compile(self, classpath, bases, sources_by_target, fingerprint): safe_mkdir(self._output_dir) compiler_classpath = self._compiler_classpath or nailgun_profile_classpath(self, self._profile) self.ng('ng-cp', *compiler_classpath) args = [ '-classpath', ':'.join(classpath), # TODO(John Sirois): untangle the jmake -C hacks somehow '-C-sourcepath', '-C%s' % ':'.join(bases), # '-sourcepath', ':'.join(bases), '-d', self._output_dir, # TODO(John Sirois): untangle this jmake specific bit '-pdb', os.path.join(self._output_dir, '%s.dependencies.pdb' % fingerprint), ] args.extend(self._args) args.extend(reduce(lambda all, sources: all | sources, sources_by_target.values())) log.debug('Executing: %s %s' % (self._main, ' '.join(args))) return self.ng(self._main, *args)
def compile(self, classpath, bases, sources): safe_mkdir(self._output_dir) compiler_classpath = self._compiler_classpath or nailgun_profile_classpath(self, self._compile_profile) self.ng("ng-cp", *compiler_classpath) # TODO(John Sirois): separate compiler profile from runtime profile args = [ "-classpath", ":".join(compiler_classpath + classpath), "-sourcepath", ":".join(bases), "-d", self._output_dir, # TODO(John Sirois): dependencyfile requires the deprecated -make:XXX - transition to ssc "-dependencyfile", self._depfile, "-make:transitivenocp", ] args.extend(self._args) args.extend(sources) self.context.log.debug("Executing: %s %s" % (self._main, " ".join(args))) return self.ng(self._main, *args)
def __init__(self, context, workdir=None): NailgunTask.__init__(self, context, workdir=context.config.get('scala-compile', 'nailgun_dir')) self._partition_size_hint = \ context.options.scala_compile_partition_size_hint \ if context.options.scala_compile_partition_size_hint != -1 else \ context.config.getint('scala-compile', 'partition_size_hint') self.check_missing_deps = context.options.scala_check_missing_deps self.check_intransitive_deps = context.options.scala_check_intransitive_deps self.check_unnecessary_deps = context.options.scala_check_unnecessary_deps if self.check_missing_deps: JvmDependencyCache.init_product_requirements(self) # We use the scala_compile_color flag if it is explicitly set on the command line. self._color = \ context.options.scala_compile_color if context.options.scala_compile_color is not None else \ context.config.getbool('scala-compile', 'color', default=True) self._compile_profile = context.config.get('scala-compile', 'compile-profile') # The target scala version. self._zinc_profile = context.config.get('scala-compile', 'zinc-profile') plugins_profile = context.config.get('scala-compile', 'scalac-plugins-profile') self._zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile) compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, self._zinc_classpath) self._zinc_jar_args = [] for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names. self._zinc_jar_args.extend(['-%s' % name, jarpath]) self._plugin_jars = nailgun_profile_classpath(self, plugins_profile) if plugins_profile else [] # All scala targets implicitly depend on the selected scala runtime. scaladeps = [] for spec in context.config.getlist('scala-compile', 'scaladeps'): scaladeps.extend(context.resolve(spec)) for target in context.targets(is_scala): target.update_dependencies(scaladeps) self._workdir = context.config.get('scala-compile', 'workdir') if workdir is None else workdir self._classes_dir = os.path.join(self._workdir, 'classes') self._analysis_cache_dir = os.path.join(self._workdir, 'analysis_cache') self._resources_dir = os.path.join(self._workdir, 'resources') self._main = context.config.get('scala-compile', 'main') self._args = context.config.getlist('scala-compile', 'args') self._jvm_args = context.config.getlist('scala-compile', 'jvm_args') if context.options.scala_compile_warnings: self._args.extend(context.config.getlist('scala-compile', 'warning_args')) else: self._args.extend(context.config.getlist('scala-compile', 'no_warning_args')) # Allow multiple flags and also comma-separated values in a single flag. plugin_names = [p for val in context.options.plugins for p in val.split(',')] \ if context.options.plugins is not None \ else context.config.getlist('scala-compile', 'scalac-plugins', default=[]) plugin_args = context.config.getdict('scala-compile', 'scalac-plugin-args', default={}) active_plugins = ScalaCompile.find_plugins(plugin_names, self._plugin_jars) for name, jar in active_plugins.items(): self._args.append('-Xplugin:%s' % jar) for arg in plugin_args.get(name, []): self._args.append('-P:%s:%s' % (name, arg)) self._confs = context.config.getlist('scala-compile', 'confs') self._depfile_dir = os.path.join(self._workdir, 'depfiles') artifact_cache_spec = context.config.getlist('scala-compile', 'artifact_caches') self.setup_artifact_cache(artifact_cache_spec)
def __init__(self, context, workdir=None): NailgunTask.__init__(self, context, workdir=context.config.get( 'scala-compile', 'nailgun_dir')) self._partition_size_hint = \ context.options.scala_compile_partition_size_hint \ if context.options.scala_compile_partition_size_hint != -1 else \ context.config.getint('scala-compile', 'partition_size_hint') # We use the scala_compile_color flag if it is explicitly set on the command line. self._color = \ context.options.scala_compile_color if context.options.scala_compile_color is not None else \ context.config.getbool('scala-compile', 'color', default=True) self._compile_profile = context.config.get( 'scala-compile', 'compile-profile') # The target scala version. self._zinc_profile = context.config.get('scala-compile', 'zinc-profile') plugins_profile = context.config.get('scala-compile', 'scalac-plugins-profile') self._zinc_classpath = nailgun_profile_classpath( self, self._zinc_profile) compiler_classpath = nailgun_profile_classpath(self, self._compile_profile) zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, self._zinc_classpath) self._zinc_jar_args = [] for (name, jarpath) in zinc_jars.items( ): # The zinc jar names are also the flag names. self._zinc_jar_args.extend(['-%s' % name, jarpath]) self._plugin_jars = nailgun_profile_classpath( self, plugins_profile) if plugins_profile else [] # All scala targets implicitly depend on the selected scala runtime. scaladeps = [] for spec in context.config.getlist('scala-compile', 'scaladeps'): scaladeps.extend(context.resolve(spec)) for target in context.targets(is_scala): target.update_dependencies(scaladeps) self._workdir = context.config.get( 'scala-compile', 'workdir') if workdir is None else workdir self._classes_dir = os.path.join(self._workdir, 'classes') self._analysis_cache_dir = os.path.join(self._workdir, 'analysis_cache') self._resources_dir = os.path.join(self._workdir, 'resources') self._main = context.config.get('scala-compile', 'main') self._args = context.config.getlist('scala-compile', 'args') self._jvm_args = context.config.getlist('scala-compile', 'jvm_args') if context.options.scala_compile_warnings: self._args.extend( context.config.getlist('scala-compile', 'warning_args')) else: self._args.extend( context.config.getlist('scala-compile', 'no_warning_args')) # Allow multiple flags and also comma-separated values in a single flag. plugin_names = [p for val in context.options.plugins for p in val.split(',')] \ if context.options.plugins is not None \ else context.config.getlist('scala-compile', 'scalac-plugins', default=[]) plugin_args = context.config.getdict('scala-compile', 'scalac-plugin-args', default={}) active_plugins = ScalaCompile.find_plugins(plugin_names, self._plugin_jars) for name, jar in active_plugins.items(): self._args.append('-Xplugin:%s' % jar) for arg in plugin_args.get(name, []): self._args.append('-P:%s:%s' % (name, arg)) self._confs = context.config.getlist('scala-compile', 'confs') self._depfile_dir = os.path.join(self._workdir, 'depfiles') artifact_cache_spec = context.config.getlist('scala-compile', 'artifact_caches') self.setup_artifact_cache(artifact_cache_spec)