def compile(self, classpath, sources, fingerprint):
safe_mkdir(self._classes_dir)
jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile)
args = [
'-classpath',
':'.join(classpath),
'-d',
self._classes_dir,
'-pdb',
os.path.join(self._classes_dir,
'%s.dependencies.pdb' % fingerprint),
]
compiler_classpath = nailgun_profile_classpath(self,
self._compiler_profile)
args.extend([
'-jcpath',
':'.join(compiler_classpath),
'-jcmainclass',
'com.twitter.common.tools.Compiler',
'-C-Tdependencyfile',
'-C%s' % self._dependencies_file,
])
args.extend(self._args)
args.extend(sources)
log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args)))
return self.runjava(_JMAKE_MAIN,
classpath=jmake_classpath,
args=args,
jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint, depfile):
jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile)
args = [
"-classpath",
":".join(classpath),
"-d",
self._classes_dir,
"-pdb",
os.path.join(self._classes_dir, "%s.dependencies.pdb" % fingerprint),
]
compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile)
args.extend(
[
"-jcpath",
":".join(compiler_classpath),
"-jcmainclass",
"com.twitter.common.tools.Compiler",
"-C-Tdependencyfile",
"-C%s" % depfile,
]
)
args.extend(self._args)
args.extend(sources)
log.debug("Executing: %s %s" % (_JMAKE_MAIN, " ".join(args)))
return self.runjava(_JMAKE_MAIN, classpath=jmake_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, output_dir, analysis_cache,
upstream_analysis_caches, depfile):
compiler_classpath = nailgun_profile_classpath(self,
self._compile_profile)
# To pass options to scalac simply prefix with -S.
args = ['-S' + x for x in self._args]
def analysis_cache_full_path(analysis_cache_product):
# We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}.
if len(analysis_cache_product) != 1:
raise TaskError(
'There can only be one analysis cache file per output directory'
)
analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems(
).next()
if len(analysis_cache_files) != 1:
raise TaskError(
'There can only be one analysis cache file per output directory'
)
return os.path.join(analysis_cache_dir, analysis_cache_files[0])
# Strings of <output dir>:<full path to analysis cache file for the classes in that dir>.
analysis_map = \
OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ])
if len(analysis_map) > 0:
args.extend([
'-analysis-map',
','.join(['%s:%s' % kv for kv in analysis_map.items()])
])
zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath,
zinc_classpath)
for (name, jarpath) in zinc_jars.items(
): # The zinc jar names are also the flag names.
args.extend(['-%s' % name, jarpath])
args.extend([
'-analysis-cache', analysis_cache, '-log-level',
self.context.options.log_level or 'info', '-classpath',
':'.join(zinc_classpath + classpath), '-output-products', depfile,
'-d', output_dir
])
if not self._color:
args.append('-no-color')
args.extend(sources)
self.context.log.debug('Executing: %s %s' %
(self._main, ' '.join(args)))
return self.runjava(self._main,
classpath=zinc_classpath,
args=args,
jvmargs=self._jvm_args)
def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile):
safe_mkdir(output_dir)
compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)
compiler_args = []
# TODO(John Sirois): separate compiler profile from runtime profile
compiler_args.extend([
# Support for outputting a dependencies file of source -> class
'-Xplugin:%s' % self.get_depemitter_plugin(),
'-P:depemitter:file:%s' % depfile
])
compiler_args.extend(self._args)
# To pass options to scalac simply prefix with -S.
args = ['-S' + x for x in compiler_args]
def analysis_cache_full_path(analysis_cache_product):
# We expect the argument to be { analysis_cache_dir, [ analysis_cache_file ]}.
if len(analysis_cache_product) != 1:
raise TaskError('There can only be one analysis cache file per output directory')
analysis_cache_dir, analysis_cache_files = analysis_cache_product.iteritems().next()
if len(analysis_cache_files) != 1:
raise TaskError('There can only be one analysis cache file per output directory')
return os.path.join(analysis_cache_dir, analysis_cache_files[0])
# Strings of <output dir>:<full path to analysis cache file for the classes in that dir>.
analysis_map = \
OrderedDict([ (k, analysis_cache_full_path(v)) for k, v in upstream_analysis_caches.itermappings() ])
if len(analysis_map) > 0:
args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in analysis_map.items()]) ])
upstream_classes_dirs = analysis_map.keys()
zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath)
for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names.
args.extend(['-%s' % name, jarpath])
args.extend([
'-analysis-cache', analysis_cache,
'-log-level', self.context.options.log_level or 'info',
'-classpath', ':'.join(zinc_classpath + classpath + upstream_classes_dirs),
'-d', output_dir
])
if not self._color:
args.append('-no-color')
args.extend(sources)
self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
return self.runjava(self._main, classpath=zinc_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, bases, sources):
safe_mkdir(self._output_dir)
compiler_classpath = (self._compiler_classpath
or nailgun_profile_classpath(
self, self._compile_profile))
self.ng('ng-cp', *compiler_classpath)
# TODO(John Sirois): separate compiler profile from runtime profile
args = [
'-classpath',
':'.join(compiler_classpath + classpath),
'-sourcepath',
':'.join(bases),
'-d',
self._output_dir,
# TODO(John Sirois): dependencyfile requires the deprecated -make:XXX - transition to ssc
'-dependencyfile',
self._depfile,
'-make:transitivenocp'
]
args.extend(self._args)
args.extend(sources)
self.context.log.debug(
'Executing: %s %s' % (self._main, ' '.join(args)))
return self.ng(self._main, *args)
def compile(self, classpath, sources):
safe_mkdir(self._classes_dir)
compiler_classpath = nailgun_profile_classpath(self,
self._compile_profile)
# TODO(John Sirois): separate compiler profile from runtime profile
args = [
'-classpath',
':'.join(compiler_classpath + classpath),
'-d',
self._classes_dir,
# Support for outputting a dependencies file of source -> class
'-Xplugin:%s' % self.get_depemitter_plugin(),
'-P:depemitter:file:%s' % self._depfile
]
args.extend(self._args)
args.extend(sources)
self.context.log.debug('Executing: %s %s' %
(self._main, ' '.join(args)))
return self.runjava(self._main,
classpath=compiler_classpath,
args=args)
def _generate_ivy_report(self):
classpath = binary_utils.nailgun_profile_classpath(self, self._profile)
reports = []
org, name = self._ivy_utils.identify()
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = self._ivy_utils.xml_report_path(conf)
out = os.path.join(self._outdir,
'%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
self.runjava('org.apache.xalan.xslt.Process',
classpath=classpath,
args=args)
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'),
self._outdir)
if self._open:
binary_utils.open(*reports)
def _generate_ivy_report(self):
classpath = binary_utils.nailgun_profile_classpath(self, self._profile)
reports = []
org, name = self._ivy_utils.identify()
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(
org=org,
name=name,
conf=conf
)
xml = self._ivy_utils.xml_report_path(conf)
out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args)
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir)
if self._open:
binary_utils.open(*reports)
def get_depemitter_plugin(self):
depemitter_classpath = nailgun_profile_classpath(self, self._depemitter_profile)
depemitter_jar = depemitter_classpath.pop()
if depemitter_classpath:
raise TaskError('Expected only 1 jar for the depemitter plugin, '
'found these extra: ' % depemitter_classpath)
return depemitter_jar
def get_depemitter_plugin(self):
depemitter_classpath = nailgun_profile_classpath(
self, self._depemitter_profile)
depemitter_jar = depemitter_classpath.pop()
if depemitter_classpath:
raise TaskError('Expected only 1 jar for the depemitter plugin, '
'found these extra: ' % depemitter_classpath)
return depemitter_jar
def compile(self, classpath, sources, output_dir, analysis_cache, upstream_analysis_caches, depfile):
safe_mkdir(output_dir)
compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)
compiler_args = []
# TODO(John Sirois): separate compiler profile from runtime profile
compiler_args.extend([
# Support for outputting a dependencies file of source -> class
'-Xplugin:%s' % self.get_depemitter_plugin(),
'-P:depemitter:file:%s' % depfile
])
compiler_args.extend(self._args)
if self._incremental:
# To pass options to scalac simply prefix with -S.
args = ['-S' + x for x in compiler_args]
if len(upstream_analysis_caches) > 0:
args.extend([ '-analysis-map', ','.join(['%s:%s' % kv for kv in upstream_analysis_caches.items()]) ])
upstream_jars = upstream_analysis_caches.keys()
zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, zinc_classpath)
for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names.
args.extend(['-%s' % name, jarpath])
args.extend([
'-analysis-cache', analysis_cache,
'-log-level', self.context.options.log_level or 'info',
'-classpath', ':'.join(zinc_classpath + classpath + upstream_jars)
])
run_classpath = zinc_classpath
else:
args = compiler_args + ['-classpath', ':'.join(compiler_classpath + classpath)]
run_classpath = compiler_classpath
args.extend([
'-d', output_dir
])
args.extend(sources)
self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
return self.runjava(self._main, classpath=run_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint, depfile):
jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile)
args = [
'-classpath', ':'.join(classpath),
'-d', self._classes_dir,
'-pdb', os.path.join(self._classes_dir, '%s.dependencies.pdb' % fingerprint),
]
compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile)
args.extend([
'-jcpath', ':'.join(compiler_classpath),
'-jcmainclass', 'com.twitter.common.tools.Compiler',
'-C-Tdependencyfile', '-C%s' % depfile,
])
args.extend(self._args)
args.extend(sources)
log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args)))
return self.runjava(_JMAKE_MAIN, classpath=jmake_classpath, args=args, jvmargs=self._jvm_args)
def compile(self, classpath, sources, fingerprint):
safe_mkdir(self._classes_dir)
jmake_classpath = nailgun_profile_classpath(self, self._jmake_profile)
self.ng('ng-cp', *jmake_classpath)
args = [
'-classpath', ':'.join(classpath),
'-d', self._classes_dir,
'-pdb', os.path.join(self._classes_dir, '%s.dependencies.pdb' % fingerprint),
]
compiler_classpath = nailgun_profile_classpath(self, self._compiler_profile)
args.extend([
'-jcpath', ':'.join(compiler_classpath),
'-jcmainclass', 'com.twitter.common.tools.Compiler',
'-C-dependencyfile', '-C%s' % self._dependencies_file
])
args.extend(self._args)
args.extend(sources)
log.debug('Executing: %s %s' % (_JMAKE_MAIN, ' '.join(args)))
return self.ng(_JMAKE_MAIN, *args)
def checkstyle(self, sources):
classpath = nailgun_profile_classpath(self, self._profile)
with self.context.state('classpath', []) as cp:
classpath.extend(jar for conf, jar in cp if conf in self._confs)
args = ['-c', self._configuration_file, '-f', 'plain']
if self._properties:
properties_file = os.path.join(self._work_dir,
'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
args.extend(['-p', properties_file])
args.extend(sources)
log.debug('Executing: %s %s' % (CHECKSTYLE_MAIN, ' '.join(args)))
return self.runjava(CHECKSTYLE_MAIN, classpath=classpath, args=args)
def compile(self, classpath, sources):
safe_mkdir(self._classes_dir)
compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)
# TODO(John Sirois): separate compiler profile from runtime profile
args = [
'-classpath', ':'.join(compiler_classpath + classpath),
'-d', self._classes_dir,
# Support for outputting a dependencies file of source -> class
'-Xplugin:%s' % self.get_depemitter_plugin(),
'-P:depemitter:file:%s' % self._depfile
]
args.extend(self._args)
args.extend(sources)
self.context.log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
return self.runjava(self._main, classpath=compiler_classpath, args=args)
def checkstyle(self, sources):
classpath = nailgun_profile_classpath(self, self._profile)
with self.context.state('classpath', []) as cp:
classpath.extend(jar for conf, jar in cp if conf in self._confs)
args = [
'-c', self._configuration_file,
'-f', 'plain'
]
if self._properties:
properties_file = os.path.join(self._work_dir, 'checkstyle.properties')
with safe_open(properties_file, 'w') as pf:
for k, v in self._properties.items():
pf.write('%s=%s\n' % (k, v))
args.extend(['-p', properties_file])
args.extend(sources)
log.debug('Executing: %s %s' % (CHECKSTYLE_MAIN, ' '.join(args)))
return self.runjava(CHECKSTYLE_MAIN, classpath=classpath, args=args)
def compile(self, classpath, bases, sources_by_target, fingerprint):
safe_mkdir(self._output_dir)
compiler_classpath = self._compiler_classpath or nailgun_profile_classpath(self, self._profile)
self.ng('ng-cp', *compiler_classpath)
args = [
'-classpath', ':'.join(classpath),
# TODO(John Sirois): untangle the jmake -C hacks somehow
'-C-sourcepath', '-C%s' % ':'.join(bases),
# '-sourcepath', ':'.join(bases),
'-d', self._output_dir,
# TODO(John Sirois): untangle this jmake specific bit
'-pdb', os.path.join(self._output_dir, '%s.dependencies.pdb' % fingerprint),
]
args.extend(self._args)
args.extend(reduce(lambda all, sources: all | sources, sources_by_target.values()))
log.debug('Executing: %s %s' % (self._main, ' '.join(args)))
return self.ng(self._main, *args)
def compile(self, classpath, bases, sources):
safe_mkdir(self._output_dir)
compiler_classpath = self._compiler_classpath or nailgun_profile_classpath(self, self._compile_profile)
self.ng("ng-cp", *compiler_classpath)
# TODO(John Sirois): separate compiler profile from runtime profile
args = [
"-classpath",
":".join(compiler_classpath + classpath),
"-sourcepath",
":".join(bases),
"-d",
self._output_dir,
# TODO(John Sirois): dependencyfile requires the deprecated -make:XXX - transition to ssc
"-dependencyfile",
self._depfile,
"-make:transitivenocp",
]
args.extend(self._args)
args.extend(sources)
self.context.log.debug("Executing: %s %s" % (self._main, " ".join(args)))
return self.ng(self._main, *args)
def __init__(self, context, workdir=None):
NailgunTask.__init__(self, context, workdir=context.config.get('scala-compile', 'nailgun_dir'))
self._partition_size_hint = \
context.options.scala_compile_partition_size_hint \
if context.options.scala_compile_partition_size_hint != -1 else \
context.config.getint('scala-compile', 'partition_size_hint')
self.check_missing_deps = context.options.scala_check_missing_deps
self.check_intransitive_deps = context.options.scala_check_intransitive_deps
self.check_unnecessary_deps = context.options.scala_check_unnecessary_deps
if self.check_missing_deps:
JvmDependencyCache.init_product_requirements(self)
# We use the scala_compile_color flag if it is explicitly set on the command line.
self._color = \
context.options.scala_compile_color if context.options.scala_compile_color is not None else \
context.config.getbool('scala-compile', 'color', default=True)
self._compile_profile = context.config.get('scala-compile', 'compile-profile') # The target scala version.
self._zinc_profile = context.config.get('scala-compile', 'zinc-profile')
plugins_profile = context.config.get('scala-compile', 'scalac-plugins-profile')
self._zinc_classpath = nailgun_profile_classpath(self, self._zinc_profile)
compiler_classpath = nailgun_profile_classpath(self, self._compile_profile)
zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath, self._zinc_classpath)
self._zinc_jar_args = []
for (name, jarpath) in zinc_jars.items(): # The zinc jar names are also the flag names.
self._zinc_jar_args.extend(['-%s' % name, jarpath])
self._plugin_jars = nailgun_profile_classpath(self, plugins_profile) if plugins_profile else []
# All scala targets implicitly depend on the selected scala runtime.
scaladeps = []
for spec in context.config.getlist('scala-compile', 'scaladeps'):
scaladeps.extend(context.resolve(spec))
for target in context.targets(is_scala):
target.update_dependencies(scaladeps)
self._workdir = context.config.get('scala-compile', 'workdir') if workdir is None else workdir
self._classes_dir = os.path.join(self._workdir, 'classes')
self._analysis_cache_dir = os.path.join(self._workdir, 'analysis_cache')
self._resources_dir = os.path.join(self._workdir, 'resources')
self._main = context.config.get('scala-compile', 'main')
self._args = context.config.getlist('scala-compile', 'args')
self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
if context.options.scala_compile_warnings:
self._args.extend(context.config.getlist('scala-compile', 'warning_args'))
else:
self._args.extend(context.config.getlist('scala-compile', 'no_warning_args'))
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = [p for val in context.options.plugins for p in val.split(',')] \
if context.options.plugins is not None \
else context.config.getlist('scala-compile', 'scalac-plugins', default=[])
plugin_args = context.config.getdict('scala-compile', 'scalac-plugin-args', default={})
active_plugins = ScalaCompile.find_plugins(plugin_names, self._plugin_jars)
for name, jar in active_plugins.items():
self._args.append('-Xplugin:%s' % jar)
for arg in plugin_args.get(name, []):
self._args.append('-P:%s:%s' % (name, arg))
self._confs = context.config.getlist('scala-compile', 'confs')
self._depfile_dir = os.path.join(self._workdir, 'depfiles')
artifact_cache_spec = context.config.getlist('scala-compile', 'artifact_caches')
self.setup_artifact_cache(artifact_cache_spec)
def __init__(self, context, workdir=None):
NailgunTask.__init__(self,
context,
workdir=context.config.get(
'scala-compile', 'nailgun_dir'))
self._partition_size_hint = \
context.options.scala_compile_partition_size_hint \
if context.options.scala_compile_partition_size_hint != -1 else \
context.config.getint('scala-compile', 'partition_size_hint')
# We use the scala_compile_color flag if it is explicitly set on the command line.
self._color = \
context.options.scala_compile_color if context.options.scala_compile_color is not None else \
context.config.getbool('scala-compile', 'color', default=True)
self._compile_profile = context.config.get(
'scala-compile', 'compile-profile') # The target scala version.
self._zinc_profile = context.config.get('scala-compile',
'zinc-profile')
plugins_profile = context.config.get('scala-compile',
'scalac-plugins-profile')
self._zinc_classpath = nailgun_profile_classpath(
self, self._zinc_profile)
compiler_classpath = nailgun_profile_classpath(self,
self._compile_profile)
zinc_jars = ScalaCompile.identify_zinc_jars(compiler_classpath,
self._zinc_classpath)
self._zinc_jar_args = []
for (name, jarpath) in zinc_jars.items(
): # The zinc jar names are also the flag names.
self._zinc_jar_args.extend(['-%s' % name, jarpath])
self._plugin_jars = nailgun_profile_classpath(
self, plugins_profile) if plugins_profile else []
# All scala targets implicitly depend on the selected scala runtime.
scaladeps = []
for spec in context.config.getlist('scala-compile', 'scaladeps'):
scaladeps.extend(context.resolve(spec))
for target in context.targets(is_scala):
target.update_dependencies(scaladeps)
self._workdir = context.config.get(
'scala-compile', 'workdir') if workdir is None else workdir
self._classes_dir = os.path.join(self._workdir, 'classes')
self._analysis_cache_dir = os.path.join(self._workdir,
'analysis_cache')
self._resources_dir = os.path.join(self._workdir, 'resources')
self._main = context.config.get('scala-compile', 'main')
self._args = context.config.getlist('scala-compile', 'args')
self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
if context.options.scala_compile_warnings:
self._args.extend(
context.config.getlist('scala-compile', 'warning_args'))
else:
self._args.extend(
context.config.getlist('scala-compile', 'no_warning_args'))
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = [p for val in context.options.plugins for p in val.split(',')] \
if context.options.plugins is not None \
else context.config.getlist('scala-compile', 'scalac-plugins', default=[])
plugin_args = context.config.getdict('scala-compile',
'scalac-plugin-args',
default={})
active_plugins = ScalaCompile.find_plugins(plugin_names,
self._plugin_jars)
for name, jar in active_plugins.items():
self._args.append('-Xplugin:%s' % jar)
for arg in plugin_args.get(name, []):
self._args.append('-P:%s:%s' % (name, arg))
self._confs = context.config.getlist('scala-compile', 'confs')
self._depfile_dir = os.path.join(self._workdir, 'depfiles')
artifact_cache_spec = context.config.getlist('scala-compile',
'artifact_caches')
self.setup_artifact_cache(artifact_cache_spec)