def __init__(self, context):
Task.__init__(self, context)
self.ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
self.confs = context.config.getlist('ivy', 'confs', default=['default'])
self.target_roots = context.target_roots
self.transitive = context.options.provides_transitive
self.workdir = context.config.get('provides', 'workdir')
self.outdir = context.options.provides_outdir or self.workdir
self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False
# Create a fake target, in case we were run directly on a JarLibrary containing nothing but JarDependencies.
# TODO(benjy): Get rid of this special-casing of jar dependencies.
context.add_new_target(self.workdir,
JvmBinary,
name='provides',
dependencies=self.target_roots,
configurations=self.confs)
context.products.require('jars')
def __init__(self, context, confs=None):
super(IvyResolve, self).__init__(context)
work_dir = context.config.get('ivy-resolve', 'workdir')
self._ivy_bootstrapper = Bootstrapper.instance()
self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
self._confs = confs or context.config.getlist('ivy-resolve', 'confs', default=['default'])
self._classpath_dir = os.path.join(work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan')
self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def setUp(self):
super(IvyUtilsGenerateIvyTest, self).setUp()
self.ivy_utils = IvyUtils(create_config(), self.create_options(), logging.Logger('test'))
class IvyUtilsGenerateIvyTest(IvyUtilsTestBase):
# TODO(John Sirois): increase coverage.
# Some examples:
# + multiple confs - via with_sources and with_docs for example
# + excludes
# + classifiers
# + with_artifact
@classmethod
def setUpClass(cls):
super(IvyUtilsGenerateIvyTest, cls).setUpClass()
cls.create_target('src/java/targets',
dedent('''
jar_library(
name='simple',
dependencies=[
jar('org1', 'name1', 'rev1'),
jar('org2', 'name2', 'rev2', force=True),
]
)
'''))
cls.simple = cls.target('src/java/targets:simple')
def setUp(self):
super(IvyUtilsGenerateIvyTest, self).setUp()
self.ivy_utils = IvyUtils(create_config(), self.create_options(), logging.Logger('test'))
def test_force_override(self):
jars = list(self.simple.dependencies)
with temporary_file_path() as ivyxml:
self.ivy_utils._generate_ivy([self.simple], jars=jars, excludes=[], ivyxml=ivyxml,
confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def find_single(self, elem, xpath):
results = list(elem.findall(xpath))
self.assertEqual(1, len(results))
return results[0]
def assert_attributes(self, elem, **kwargs):
self.assertEqual(dict(**kwargs), dict(elem.attrib))
class IvyResolve(NailgunTask):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
NailgunTask.setup_parser(option_group, args, mkflag)
flag = mkflag('override')
option_group.add_option(flag, action='append', dest='ivy_resolve_overrides',
help="""Specifies a jar dependency override in the form:
[org]#[name]=(revision|url)
For example, to specify 2 overrides:
%(flag)s=com.foo#bar=0.1.2 \\
%(flag)s=com.baz#spam=file:///tmp/spam.jar
""" % dict(flag=flag))
report = mkflag("report")
option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report",
action="callback", callback=mkflag.set_bool, default=False,
help = "[%default] Generate an ivy resolve html report")
option_group.add_option(mkflag("open"), mkflag("open", negate=True),
dest="ivy_resolve_open", default=False,
action="callback", callback=mkflag.set_bool,
help="[%%default] Attempt to open the generated ivy resolve report "
"in a browser (implies %s)." % report)
option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir",
help="Emit ivy report outputs in to this directory.")
option_group.add_option(mkflag("args"), dest="ivy_args", action="append", default=[],
help = "Pass these extra args to ivy.")
option_group.add_option(mkflag("mutable-pattern"), dest="ivy_mutable_pattern",
help="If specified, all artifact revisions matching this pattern will "
"be treated as mutable unless a matching artifact explicitly "
"marks mutable as False.")
def __init__(self, context, confs=None):
super(IvyResolve, self).__init__(context)
work_dir = context.config.get('ivy-resolve', 'workdir')
self._ivy_bootstrapper = Bootstrapper.instance()
self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
self._confs = confs or context.config.getlist('ivy-resolve', 'confs', default=['default'])
self._classpath_dir = os.path.join(work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan')
self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def invalidate_for(self):
return self.context.options.ivy_resolve_overrides
def execute(self, targets):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
groups = self.context.products.get_data('exclusives_groups')
executor = self.create_java_executor()
# Below, need to take the code that actually execs ivy, and invoke it once for each
# group. Then after running ivy, we need to take the resulting classpath, and load it into
# the build products.
# The set of groups we need to consider is complicated:
# - If there are no conflicting exclusives (ie, there's only one entry in the map),
# then we just do the one.
# - If there are conflicts, then there will be at least three entries in the groups map:
# - the group with no exclusives (X)
# - the two groups that are in conflict (A and B).
# In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
# because things in A and B can depend on things in X; and so they can indirectly depend
# on the dependencies of X.
# (I think this well be covered by the computed transitive dependencies of
# A and B. But before pushing this change, review this comment, and make sure that this is
# working correctly.)
for group_key in groups.get_group_keys():
# Narrow the groups target set to just the set of targets that we're supposed to build.
# Normally, this shouldn't be different from the contents of the group.
group_targets = groups.get_targets_for_group_key(group_key) & set(targets)
# NOTE(pl): The symlinked ivy.xml (for IDEs, particularly IntelliJ) in the presence of
# multiple exclusives groups will end up as the last exclusives group run. I'd like to
# deprecate this eventually, but some people rely on it, and it's not clear to me right now
# whether telling them to use IdeaGen instead is feasible.
classpath = self.ivy_resolve(group_targets,
executor=executor,
symlink_ivyxml=True,
workunit_name='ivy-resolve')
if self.context.products.is_required_data('ivy_jar_products'):
self._populate_ivy_jar_products(group_targets)
for conf in self._confs:
# It's important we add the full classpath as an (ordered) unit for code that is classpath
# order sensitive
classpath_entries = map(lambda entry: (conf, entry), classpath)
groups.update_compatible_classpaths(group_key, classpath_entries)
if self._report:
self._generate_ivy_report(group_targets)
create_jardeps_for = self.context.products.isrequired('jar_dependencies')
if create_jardeps_for:
genmap = self.context.products.get('jar_dependencies')
for target in filter(create_jardeps_for, targets):
self._ivy_utils.mapjars(genmap, target, executor=executor,
workunit_factory=self.context.new_workunit)
def check_artifact_cache_for(self, invalidation_check):
# Ivy resolution is an output dependent on the entire target set, and is not divisible
# by target. So we can only cache it keyed by the entire target set.
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
return [global_vts]
def _populate_ivy_jar_products(self, targets):
"""Populate the build products with an IvyInfo object for each generated ivy report."""
ivy_products = self.context.products.get_data('ivy_jar_products') or defaultdict(list)
for conf in self._confs:
ivyinfo = self._ivy_utils.parse_xml_report(targets, conf)
if ivyinfo:
# Value is a list, to accommodate multiple exclusives groups.
ivy_products[conf].append(ivyinfo)
self.context.products.safe_create_data('ivy_jar_products', lambda: ivy_products)
def _generate_ivy_report(self, targets):
def make_empty_report(report, organisation, module, conf):
no_deps_xml_template = """
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?>
<ivy-report version="1.0">
<info
organisation="%(organisation)s"
module="%(module)s"
revision="latest.integration"
conf="%(conf)s"
confs="%(conf)s"
date="%(timestamp)s"/>
</ivy-report>
"""
no_deps_xml = no_deps_xml_template % dict(organisation=organisation,
module=module,
conf=conf,
timestamp=time.strftime('%Y%m%d%H%M%S'))
with open(report, 'w') as report_handle:
print(no_deps_xml, file=report_handle)
classpath = self._jvm_tool_bootstrapper.get_jvm_tool_classpath(self._ivy_bootstrap_key,
self.create_java_executor())
reports = []
org, name = self._ivy_utils.identify(targets)
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
# Xalan needs this dir to exist - ensure that, but do no more - we have no clue where this
# points.
safe_mkdir(self._outdir, clean=False)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = self._ivy_utils.xml_report_path(targets, conf)
if not os.path.exists(xml):
make_empty_report(xml, org, name, conf)
out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
if 0 != self.runjava(classpath=classpath, main='org.apache.xalan.xslt.Process',
args=args, workunit_name='report'):
raise TaskError
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir)
if self._open:
binary_util.ui_open(*reports)
def ivy_resolve(self, targets, executor=None, symlink_ivyxml=False, silent=False,
workunit_name=None, workunit_labels=None):
if executor and not isinstance(executor, Executor):
raise ValueError('The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
targets = set(targets)
if not targets:
return []
work_dir = self.context.config.get('ivy-resolve', 'workdir')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
with self.invalidated(targets,
only_buildfiles=True,
invalidate_dependents=True,
silent=silent) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(work_dir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with Task.symlink_map_lock:
all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
class Provides(Task):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
option_group.add_option(mkflag("outdir"), dest="provides_outdir",
help="Emit provides outputs into this directory.")
option_group.add_option(mkflag("transitive"), default=False,
action="store_true", dest='provides_transitive',
help="Shows the symbols provided not just by the specified targets but by all their transitive dependencies.")
option_group.add_option(mkflag("also-write-to-stdout"), default=False,
action="store_true", dest='provides_also_write_to_stdout',
help="If set, also outputs the provides information to stdout.")
def __init__(self, context):
Task.__init__(self, context)
self.ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
self.confs = context.config.getlist('ivy', 'confs', default=['default'])
self.target_roots = context.target_roots
self.transitive = context.options.provides_transitive
self.workdir = context.config.get('provides', 'workdir')
self.outdir = context.options.provides_outdir or self.workdir
self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False
# Create a fake target, in case we were run directly on a JarLibrary containing nothing but JarDependencies.
# TODO(benjy): Get rid of this special-casing of jar dependencies.
context.add_new_target(self.workdir,
JvmBinary,
name='provides',
dependencies=self.target_roots,
configurations=self.confs)
context.products.require('jars')
def execute(self, targets):
for conf in self.confs:
outpath = os.path.join(self.outdir, '%s.%s.provides' %
(self.ivy_utils.identify(targets)[1], conf))
if self.transitive:
outpath += '.transitive'
ivyinfo = self.ivy_utils.parse_xml_report(self.context, conf)
jar_paths = OrderedSet()
for root in self.target_roots:
jar_paths.update(self.get_jar_paths(ivyinfo, root, conf))
with open(outpath, 'w') as outfile:
def do_write(s):
outfile.write(s)
if self.also_write_to_stdout:
sys.stdout.write(s)
for jar in jar_paths:
do_write('# from jar %s\n' % jar)
for line in self.list_jar(jar):
if line.endswith('.class'):
class_name = line[:-6].replace('/', '.')
do_write(class_name)
do_write('\n')
print('Wrote provides information to %s' % outpath)
def get_jar_paths(self, ivyinfo, target, conf):
jar_paths = OrderedSet()
if target.is_jar_library:
# Jar library proxies jar dependencies or jvm targets, so the jars are just those of the
# dependencies.
for paths in [ self.get_jar_paths(ivyinfo, dep, conf) for dep in target.dependencies ]:
jar_paths.update(paths)
elif target.is_jar_dependency:
ref = IvyModuleRef(target.org, target.name, target.rev, conf)
jar_paths.update(self.get_jar_paths_for_ivy_module(ivyinfo, ref))
elif target.is_jvm:
for basedir, jars in self.context.products.get('jars').get(target).items():
jar_paths.update([os.path.join(basedir, jar) for jar in jars])
if self.transitive:
for dep in target.dependencies:
jar_paths.update(self.get_jar_paths(ivyinfo, dep, conf))
return jar_paths
def get_jar_paths_for_ivy_module(self, ivyinfo, ref):
jar_paths = OrderedSet()
module = ivyinfo.modules_by_ref[ref]
jar_paths.update([a.path for a in module.artifacts])
if self.transitive:
for dep in ivyinfo.deps_by_caller.get(ref, []):
jar_paths.update(self.get_jar_paths_for_ivy_module(ivyinfo, dep))
return jar_paths
def list_jar(self, path):
with open_jar(path, 'r') as jar:
return jar.namelist()
