class IvyResolve(NailgunTask):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
NailgunTask.setup_parser(option_group, args, mkflag)
flag = mkflag('override')
option_group.add_option(
flag,
action='append',
dest='ivy_resolve_overrides',
help='''Specifies a jar dependency override in the form:
[org]#[name]=(revision|url)
For example, to specify 2 overrides:
%(flag)s=com.foo#bar=0.1.2 \\
%(flag)s=com.baz#spam=file:///tmp/spam.jar
''' % dict(flag=flag))
report = mkflag("report")
option_group.add_option(
report,
mkflag("report", negate=True),
dest="ivy_resolve_report",
action="callback",
callback=mkflag.set_bool,
default=False,
help="[%default] Generate an ivy resolve html report")
option_group.add_option(
mkflag("open"),
mkflag("open", negate=True),
dest="ivy_resolve_open",
default=False,
action="callback",
callback=mkflag.set_bool,
help="[%%default] Attempt to open the generated ivy resolve report "
"in a browser (implies %s)." % report)
option_group.add_option(
mkflag("outdir"),
dest="ivy_resolve_outdir",
help="Emit ivy report outputs in to this directory.")
def __init__(self, context):
classpath = context.config.getlist('ivy', 'classpath')
nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir')
NailgunTask.__init__(self,
context,
classpath=classpath,
workdir=nailgun_dir)
self._ivy_settings = context.config.get('ivy', 'ivy_settings')
self._cachedir = context.config.get('ivy', 'cache_dir')
self._confs = context.config.getlist('ivy-resolve', 'confs')
self._transitive = context.config.getbool('ivy-resolve', 'transitive')
self._args = context.config.getlist('ivy-resolve', 'args')
self._profile = context.config.get('ivy-resolve', 'profile')
self._template_path = os.path.join('ivy_resolve', 'ivy.mk')
self._work_dir = context.config.get('ivy-resolve', 'workdir')
self._classpath_file = os.path.join(self._work_dir, 'classpath')
self._classpath_dir = os.path.join(self._work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(
self._work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_utils = IvyUtils(context, self._cachedir)
def parse_override(override):
match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override)
if not match:
raise TaskError('Invalid dependency override: %s' % override)
org, name, rev_or_url = match.groups()
def fmt_message(message, template):
return message % dict(
overridden='%s#%s;%s' %
(template.org, template.module, template.version),
rev=rev_or_url,
url=rev_or_url)
def replace_rev(template):
context.log.info(
fmt_message('Overrode %(overridden)s with rev %(rev)s',
template))
return template.extend(version=rev_or_url,
url=None,
force=True)
def replace_url(template):
context.log.info(
fmt_message(
'Overrode %(overridden)s with snapshot at %(url)s',
template))
return template.extend(version='SNAPSHOT',
url=rev_or_url,
force=True)
replace = replace_url if re.match(r'^\w+://.+',
rev_or_url) else replace_rev
return (org, name), replace
self._overrides = {}
if context.options.ivy_resolve_overrides:
self._overrides.update(
parse_override(o)
for o in context.options.ivy_resolve_overrides)
def invalidate_for(self):
return self.context.options.ivy_resolve_overrides
def execute(self, targets):
"""
Resolves the specified confs for the configured targets and returns an iterator over tuples
of (conf, jar path).
"""
def dirname_for_requested_targets(targets):
"""Where we put the classpath file for this set of targets."""
sha = hashlib.sha1()
for t in targets:
sha.update(t.id)
return sha.hexdigest()
def is_classpath(t):
return is_internal(t) and any(
jar for jar in t.jar_dependencies if jar.rev)
target_workdir = os.path.join(self._work_dir,
dirname_for_requested_targets(targets))
target_classpath_file = os.path.join(target_workdir, 'classpath')
with self.changed(filter(is_classpath, targets),
only_buildfiles=True) as changed_deps:
if changed_deps:
self._exec_ivy(
target_workdir, targets,
['-cachepath', target_classpath_file, '-confs'] +
self._confs)
if os.path.exists(target_classpath_file):
def safe_link(src, dest):
if os.path.exists(dest):
os.unlink(dest)
os.symlink(src, dest)
# Symlink to the current classpath file.
safe_link(target_classpath_file, self._classpath_file)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
target_ivyxml = os.path.join(target_workdir, 'ivy.xml')
safe_link(target_ivyxml, ivyxml_symlink)
if os.path.exists(self._classpath_file):
with self._cachepath(self._classpath_file) as classpath:
with self.context.state('classpath', []) as cp:
for path in classpath:
if self._is_jar(path):
for conf in self._confs:
cp.append((conf, path.strip()))
if self._report:
self._generate_ivy_report()
create_jardeps_for = self.context.products.isrequired(
'jar_dependencies')
if create_jardeps_for:
genmap = self.context.products.get('jar_dependencies')
for target in filter(create_jardeps_for, targets):
self._mapjars(genmap, target)
def _generate_ivy(self, jars, excludes, ivyxml):
org, name = self._ivy_utils.identify()
template_data = TemplateData(
org=org,
module=name,
version='latest.integration',
publications=None,
dependencies=[self._generate_jar_template(jar) for jar in jars],
excludes=[
self._generate_exclude_template(exclude)
for exclude in excludes
])
safe_mkdir(os.path.dirname(ivyxml))
with open(ivyxml, 'w') as output:
generator = Generator(pkgutil.get_data(__name__,
self._template_path),
root_dir=get_buildroot(),
lib=template_data)
generator.write(output)
def _generate_ivy_report(self):
classpath = binary_utils.nailgun_profile_classpath(self, self._profile)
reports = []
org, name = self._ivy_utils.identify()
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = self._ivy_utils.xml_report_path(conf)
out = os.path.join(self._outdir,
'%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
self.runjava('org.apache.xalan.xslt.Process',
classpath=classpath,
args=args)
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'),
self._outdir)
if self._open:
binary_utils.open(*reports)
def _calculate_classpath(self, targets):
jars = set()
excludes = set()
def collect_jars(target):
if target.jar_dependencies:
jars.update(jar for jar in target.jar_dependencies if jar.rev)
if target.excludes:
excludes.update(target.excludes)
for target in targets:
target.walk(collect_jars, is_jvm)
return jars, excludes
def _generate_jar_template(self, jar):
template = TemplateData(
org=jar.org,
module=jar.name,
version=jar.rev,
force=jar.force,
excludes=[
self._generate_exclude_template(exclude)
for exclude in jar.excludes
],
transitive=jar.transitive,
ext=jar.ext,
url=jar.url,
configurations=';'.join(jar._configurations),
)
override = self._overrides.get((jar.org, jar.name))
return override(template) if override else template
def _generate_exclude_template(self, exclude):
return TemplateData(org=exclude.org, name=exclude.name)
@contextmanager
def _cachepath(self, file):
with safe_open(file, 'r') as cp:
yield (path.strip() for path in cp.read().split(os.pathsep)
if path.strip())
def _mapjars(self, genmap, target):
mapdir = os.path.join(self._classpath_dir, target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve',
'%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
mapdir,
'-symlink',
'-confs',
]
ivyargs.extend(target.configurations or self._confs)
self._exec_ivy(mapdir, [target], ivyargs)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for file in os.listdir(confdir):
if self._is_jar(file):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(file)
genmap.add((org, name),
confdir).append(file)
genmap.add(target, confdir).append(file)
genmap.add((target, conf),
confdir).append(file)
genmap.add((org, name, conf),
confdir).append(file)
def _is_jar(self, path):
return path.endswith('.jar')
def _exec_ivy(self, target_workdir, targets, args):
ivyxml = os.path.join(target_workdir, 'ivy.xml')
jars, excludes = self._calculate_classpath(targets)
self._generate_ivy(jars, excludes, ivyxml)
ivy_args = [
'-settings',
self._ivy_settings,
'-cache',
self._cachedir,
'-ivy',
ivyxml,
]
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
result = self.runjava('org.apache.ivy.Main', args=ivy_args)
if result != 0:
raise TaskError('org.apache.ivy.Main returned %d' % result)
class IvyResolve(NailgunTask):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
NailgunTask.setup_parser(option_group, args, mkflag)
flag = mkflag('override')
option_group.add_option(flag, action='append', dest='ivy_resolve_overrides',
help="""Specifies a jar dependency override in the form:
[org]#[name]=(revision|url)
For example, to specify 2 overrides:
%(flag)s=com.foo#bar=0.1.2 \\
%(flag)s=com.baz#spam=file:///tmp/spam.jar
""" % dict(flag=flag))
report = mkflag("report")
option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report",
action="callback", callback=mkflag.set_bool, default=False,
help = "[%default] Generate an ivy resolve html report")
option_group.add_option(mkflag("open"), mkflag("open", negate=True),
dest="ivy_resolve_open", default=False,
action="callback", callback=mkflag.set_bool,
help="[%%default] Attempt to open the generated ivy resolve report "
"in a browser (implies %s)." % report)
option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir",
help="Emit ivy report outputs in to this directory.")
option_group.add_option(mkflag("cache"), dest="ivy_resolve_cache",
help="Use this directory as the ivy cache, instead of the "
"default specified in pants.ini.")
option_group.add_option(mkflag("mutable-pattern"), dest="ivy_mutable_pattern",
help="If specified, all artifact revisions matching this pattern will "
"be treated as mutable unless a matching artifact explicitly "
"marks mutable as False.")
def __init__(self, context, confs=None):
nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir')
NailgunTask.__init__(self, context, workdir=nailgun_dir)
self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir')
self._confs = confs or context.config.getlist('ivy-resolve', 'confs')
self._work_dir = context.config.get('ivy-resolve', 'workdir')
self._classpath_dir = os.path.join(self._work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan')
self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def invalidate_for(self):
return self.context.options.ivy_resolve_overrides
def execute(self, targets):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
groups = self.context.products.get_data('exclusives_groups')
# Below, need to take the code that actually execs ivy, and invoke it once for each
# group. Then after running ivy, we need to take the resulting classpath, and load it into
# the build products.
# The set of groups we need to consider is complicated:
# - If there are no conflicting exclusives (ie, there's only one entry in the map),
# then we just do the one.
# - If there are conflicts, then there will be at least three entries in the groups map:
# - the group with no exclusives (X)
# - the two groups that are in conflict (A and B).
# In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
# because things in A and B can depend on things in X; and so they can indirectly depend
# on the dependencies of X.
# (I think this well be covered by the computed transitive dependencies of
# A and B. But before pushing this change, review this comment, and make sure that this is
# working correctly.)
for group_key in groups.get_group_keys():
# Narrow the groups target set to just the set of targets that we're supposed to build.
# Normally, this shouldn't be different from the contents of the group.
group_targets = groups.get_targets_for_group_key(group_key) & set(targets)
# NOTE(pl): The symlinked ivy.xml (for IDEs, particularly IntelliJ) in the presence of
# multiple exclusives groups will end up as the last exclusives group run. I'd like to
# deprecate this eventually, but some people rely on it, and it's not clear to me right now
# whether telling them to use IdeaGen instead is feasible.
classpath = self.ivy_resolve(group_targets,
java_runner=self.runjava_indivisible,
symlink_ivyxml=True)
if self.context.products.isrequired('ivy_jar_products'):
self._populate_ivy_jar_products(group_targets)
for conf in self._confs:
for path in classpath:
groups.update_compatible_classpaths(group_key, [(conf, path)])
if self._report:
self._generate_ivy_report(group_targets)
create_jardeps_for = self.context.products.isrequired(self._ivy_utils._mapfor_typename())
if create_jardeps_for:
genmap = self.context.products.get(self._ivy_utils._mapfor_typename())
for target in filter(create_jardeps_for, targets):
self._ivy_utils.mapjars(genmap, target, java_runner=self.runjava_indivisible)
def check_artifact_cache_for(self, invalidation_check):
# Ivy resolution is an output dependent on the entire target set, and is not divisible
# by target. So we can only cache it keyed by the entire target set.
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
return [global_vts]
def _populate_ivy_jar_products(self, targets):
"""Populate the build products with an IvyInfo object for each generated ivy report."""
ivy_products = self.context.products.get_data('ivy_jar_products') or defaultdict(list)
for conf in self._confs:
ivyinfo = self._ivy_utils.parse_xml_report(targets, conf)
if ivyinfo:
ivy_products[conf].append(ivyinfo) # Value is a list, to accommodate multiple exclusives groups.
self.context.products.set_data('ivy_jar_products', ivy_products)
def _generate_ivy_report(self, targets):
def make_empty_report(report, organisation, module, conf):
no_deps_xml_template = """
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?>
<ivy-report version="1.0">
<info
organisation="%(organisation)s"
module="%(module)s"
revision="latest.integration"
conf="%(conf)s"
confs="%(conf)s"
date="%(timestamp)s"/>
</ivy-report>
"""
no_deps_xml = no_deps_xml_template % dict(organisation=organisation,
module=module,
conf=conf,
timestamp=time.strftime('%Y%m%d%H%M%S'))
with open(report, 'w') as report_handle:
print(no_deps_xml, file=report_handle)
classpath = self._jvm_tool_bootstrapper.get_jvm_tool_classpath(self._ivy_bootstrap_key,
self.runjava_indivisible)
reports = []
org, name = self._ivy_utils.identify(targets)
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = self._ivy_utils.xml_report_path(targets, conf)
if not os.path.exists(xml):
make_empty_report(xml, org, name, conf)
out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process', classpath=classpath,
args=args, workunit_name='report'):
raise TaskError
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir)
if self._open:
binary_util.ui_open(*reports)
class IvyResolve(NailgunTask):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
NailgunTask.setup_parser(option_group, args, mkflag)
flag = mkflag('override')
option_group.add_option(flag, action='append', dest='ivy_resolve_overrides',
help='''Specifies a jar dependency override in the form:
[org]#[name]=(revision|url)
For example, to specify 2 overrides:
%(flag)s=com.foo#bar=0.1.2 \\
%(flag)s=com.baz#spam=file:///tmp/spam.jar
''' % dict(flag=flag))
report = mkflag("report")
option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report",
action="callback", callback=mkflag.set_bool, default=False,
help = "[%default] Generate an ivy resolve html report")
option_group.add_option(mkflag("open"), mkflag("open", negate=True),
dest="ivy_resolve_open", default=False,
action="callback", callback=mkflag.set_bool,
help="[%%default] Attempt to open the generated ivy resolve report "
"in a browser (implies %s)." % report)
option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir",
help="Emit ivy report outputs in to this directory.")
option_group.add_option(mkflag("cache"), dest="ivy_resolve_cache",
help="Use this directory as the ivy cache, instead of the " \
"default specified in pants.ini.")
def __init__(self, context):
classpath = context.config.getlist('ivy', 'classpath')
nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir')
NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir)
self._ivy_settings = context.config.get('ivy', 'ivy_settings')
self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir')
self._confs = context.config.getlist('ivy-resolve', 'confs')
self._transitive = context.config.getbool('ivy-resolve', 'transitive')
self._args = context.config.getlist('ivy-resolve', 'args')
self._profile = context.config.get('ivy-resolve', 'profile')
self._template_path = os.path.join('ivy_resolve', 'ivy.mustache')
self._work_dir = context.config.get('ivy-resolve', 'workdir')
self._classpath_file = os.path.join(self._work_dir, 'classpath')
self._classpath_dir = os.path.join(self._work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_utils = IvyUtils(context, self._cachedir)
def parse_override(override):
match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override)
if not match:
raise TaskError('Invalid dependency override: %s' % override)
org, name, rev_or_url = match.groups()
def fmt_message(message, template):
return message % dict(
overridden='%s#%s;%s' % (template.org, template.module, template.version),
rev=rev_or_url,
url=rev_or_url
)
def replace_rev(template):
context.log.info(fmt_message('Overrode %(overridden)s with rev %(rev)s', template))
return template.extend(version=rev_or_url, url=None, force=True)
def replace_url(template):
context.log.info(fmt_message('Overrode %(overridden)s with snapshot at %(url)s', template))
return template.extend(version='SNAPSHOT', url=rev_or_url, force=True)
replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev
return (org, name), replace
self._overrides = {}
if context.options.ivy_resolve_overrides:
self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides)
def invalidate_for(self):
return self.context.options.ivy_resolve_overrides
def execute(self, targets):
"""
Resolves the specified confs for the configured targets and returns an iterator over tuples
of (conf, jar path).
"""
def dirname_for_requested_targets(targets):
"""Where we put the classpath file for this set of targets."""
sha = hashlib.sha1()
for t in targets:
sha.update(t.id)
return sha.hexdigest()
def is_classpath(t):
return is_internal(t) and any(jar for jar in t.jar_dependencies if jar.rev)
classpath_targets = filter(is_classpath, targets)
target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(targets))
target_classpath_file = os.path.join(target_workdir, 'classpath')
with self.invalidated(classpath_targets, only_buildfiles=True, invalidate_dependants=True) as invalidation_check:
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously build a superset of targets.
if len(invalidation_check.invalid_vts) > 0 or not os.path.exists(target_classpath_file):
self._exec_ivy(target_workdir, targets, [
'-cachepath', target_classpath_file,
'-confs'
] + self._confs)
if not os.path.exists(target_classpath_file):
raise TaskError, 'Ivy failed to create classpath file at %s' % target_classpath_file
def safe_link(src, dest):
if os.path.exists(dest):
os.unlink(dest)
os.symlink(src, dest)
# Symlink to the current classpath file.
safe_link(target_classpath_file, self._classpath_file)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
target_ivyxml = os.path.join(target_workdir, 'ivy.xml')
safe_link(target_ivyxml, ivyxml_symlink)
with self._cachepath(self._classpath_file) as classpath:
with self.context.state('classpath', []) as cp:
for path in classpath:
if self._is_jar(path):
for conf in self._confs:
cp.append((conf, path.strip()))
if self._report:
self._generate_ivy_report()
create_jardeps_for = self.context.products.isrequired('jar_dependencies')
if create_jardeps_for:
genmap = self.context.products.get('jar_dependencies')
for target in filter(create_jardeps_for, targets):
self._mapjars(genmap, target)
def _generate_ivy(self, jars, excludes, ivyxml):
org, name = self._ivy_utils.identify()
template_data = TemplateData(
org=org,
module=name,
version='latest.integration',
publications=None,
dependencies=[self._generate_jar_template(jar) for jar in jars],
excludes=[self._generate_exclude_template(exclude) for exclude in excludes]
)
safe_mkdir(os.path.dirname(ivyxml))
with open(ivyxml, 'w') as output:
generator = Generator(pkgutil.get_data(__name__, self._template_path),
root_dir = get_buildroot(),
lib = template_data)
generator.write(output)
def _generate_ivy_report(self):
classpath = binary_utils.nailgun_profile_classpath(self, self._profile)
reports = []
org, name = self._ivy_utils.identify()
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(
org=org,
name=name,
conf=conf
)
xml = self._ivy_utils.xml_report_path(conf)
out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args)
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir)
if self._open:
binary_utils.open(*reports)
def _calculate_classpath(self, targets):
jars = set()
excludes = set()
def collect_jars(target):
if target.jar_dependencies:
jars.update(jar for jar in target.jar_dependencies if jar.rev)
if target.excludes:
excludes.update(target.excludes)
for target in targets:
target.walk(collect_jars, is_jvm)
return jars, excludes
def _generate_jar_template(self, jar):
template = TemplateData(
org = jar.org,
module = jar.name,
version = jar.rev,
force = jar.force,
excludes = [self._generate_exclude_template(exclude) for exclude in jar.excludes],
transitive = jar.transitive,
artifacts = jar.artifacts,
configurations = ';'.join(jar._configurations),
)
override = self._overrides.get((jar.org, jar.name))
return override(template) if override else template
def _generate_exclude_template(self, exclude):
return TemplateData(org = exclude.org, name = exclude.name)
@contextmanager
def _cachepath(self, file):
with safe_open(file, 'r') as cp:
yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip())
def _mapjars(self, genmap, target):
"""
Parameters:
genmap: the jar_dependencies ProductMapping entry for the required products.
target: the target whose jar dependencies are being retrieved.
"""
mapdir = os.path.join(self._classpath_dir, target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve', '%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
'-symlink',
'-confs',
]
ivyargs.extend(target.configurations or self._confs)
self._exec_ivy(mapdir, [target], ivyargs)
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for file in os.listdir(confdir):
if self._is_jar(file):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(file)
genmap.add((org, name), confdir).append(file)
genmap.add(target, confdir).append(file)
genmap.add((target, conf), confdir).append(file)
genmap.add((org, name, conf), confdir).append(file)
def _is_jar(self, path):
return path.endswith('.jar')
def _exec_ivy(self, target_workdir, targets, args):
ivyxml = os.path.join(target_workdir, 'ivy.xml')
jars, excludes = self._calculate_classpath(targets)
self._generate_ivy(jars, excludes, ivyxml)
ivy_args = [
'-settings', self._ivy_settings,
'-cache', self._cachedir,
'-ivy', ivyxml,
]
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
result = self.runjava('org.apache.ivy.Main', args=ivy_args)
if result != 0:
raise TaskError('org.apache.ivy.Main returned %d' % result)
class IvyResolve(NailgunTask):
@classmethod
def setup_parser(cls, option_group, args, mkflag):
NailgunTask.setup_parser(option_group, args, mkflag)
flag = mkflag('override')
option_group.add_option(
flag,
action='append',
dest='ivy_resolve_overrides',
help='''Specifies a jar dependency override in the form:
[org]#[name]=(revision|url)
For example, to specify 2 overrides:
%(flag)s=com.foo#bar=0.1.2 \\
%(flag)s=com.baz#spam=file:///tmp/spam.jar
''' % dict(flag=flag))
report = mkflag("report")
option_group.add_option(
report,
mkflag("report", negate=True),
dest="ivy_resolve_report",
action="callback",
callback=mkflag.set_bool,
default=False,
help="[%default] Generate an ivy resolve html report")
option_group.add_option(
mkflag("open"),
mkflag("open", negate=True),
dest="ivy_resolve_open",
default=False,
action="callback",
callback=mkflag.set_bool,
help="[%%default] Attempt to open the generated ivy resolve report "
"in a browser (implies %s)." % report)
option_group.add_option(
mkflag("outdir"),
dest="ivy_resolve_outdir",
help="Emit ivy report outputs in to this directory.")
option_group.add_option(
mkflag("cache"),
dest="ivy_resolve_cache",
help="Use this directory as the ivy cache, instead of the "
"default specified in pants.ini.")
option_group.add_option(mkflag("args"),
dest="ivy_args",
action="append",
default=[],
help="Pass these extra args to ivy.")
option_group.add_option(
mkflag("mutable-pattern"),
dest="ivy_mutable_pattern",
help=
"If specified, all artifact revisions matching this pattern will "
"be treated as mutable unless a matching artifact explicitly "
"marks mutable as False.")
def __init__(self, context, confs=None):
classpath = context.config.getlist('ivy', 'classpath')
nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir')
NailgunTask.__init__(self,
context,
classpath=classpath,
workdir=nailgun_dir)
self._cachedir = context.options.ivy_resolve_cache or context.config.get(
'ivy', 'cache_dir')
self._confs = confs or context.config.getlist('ivy-resolve', 'confs')
self._work_dir = context.config.get('ivy-resolve', 'workdir')
self._classpath_dir = os.path.join(self._work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(
self._work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve',
'bootstrap-tools',
':xalan')
self._bootstrap_utils.register_jvm_build_tools(self._ivy_bootstrap_key,
ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
artifact_cache_spec = context.config.getlist('ivy-resolve',
'artifact_caches',
default=[])
self.setup_artifact_cache(artifact_cache_spec)
def invalidate_for(self):
return self.context.options.ivy_resolve_overrides
def execute(self, targets):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
groups = self.context.products.get_data('exclusives_groups')
# Below, need to take the code that actually execs ivy, and invoke it once for each
# group. Then after running ivy, we need to take the resulting classpath, and load it into
# the build products.
# The set of groups we need to consider is complicated:
# - If there are no conflicting exclusives (ie, there's only one entry in the map),
# then we just do the one.
# - If there are conflicts, then there will be at least three entries in the groups map:
# - the group with no exclusives (X)
# - the two groups that are in conflict (A and B).
# In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
# because things in A and B can depend on things in X; and so they can indirectly depend
# on the dependencies of X.
# (I think this well be covered by the computed transitive dependencies of
# A and B. But before pushing this change, review this comment, and make sure that this is
# working correctly.)
for group_key in groups.get_group_keys():
# Narrow the groups target set to just the set of targets that we're supposed to build.
# Normally, this shouldn't be different from the contents of the group.
group_targets = groups.get_targets_for_group_key(group_key) & set(
targets)
# NOTE(pl): The symlinked ivy.xml (for IDEs, particularly IntelliJ) in the presence of
# multiple exclusives groups will end up as the last exclusives group run. I'd like to
# deprecate this eventually, but some people rely on it, and it's not clear to me right now
# whether telling them to use IdeaGen instead is feasible.
classpath = self.ivy_resolve(group_targets,
java_runner=self.runjava_indivisible,
symlink_ivyxml=True)
for conf in self._confs:
for path in classpath:
groups.update_compatible_classpaths(
group_key, [(conf, path)])
if self._report:
self._generate_ivy_report(group_targets)
if self.context.products.isrequired('ivy_jar_products'):
self._populate_ivy_jar_products(targets)
create_jardeps_for = self.context.products.isrequired(
self._ivy_utils._mapfor_typename())
if create_jardeps_for:
genmap = self.context.products.get(
self._ivy_utils._mapfor_typename())
for target in filter(create_jardeps_for, targets):
self._ivy_utils.mapjars(genmap,
target,
java_runner=self.runjava_indivisible)
def check_artifact_cache_for(self, invalidation_check):
# Ivy resolution is an output dependent on the entire target set, and is not divisible
# by target. So we can only cache it keyed by the entire target set.
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
return [global_vts]
def _populate_ivy_jar_products(self, targets):
"""Populate the build products with an IvyInfo object for each generated ivy report."""
ivy_products = {}
for conf in self._confs:
ivyinfo = self._ivy_utils.parse_xml_report(targets, conf)
if ivyinfo:
ivy_products[conf] = ivyinfo
self.context.products.set_data('ivy_jar_products', ivy_products)
def _generate_ivy_report(self, targets):
def make_empty_report(report, organisation, module, conf):
no_deps_xml_template = """
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?>
<ivy-report version="1.0">
<info
organisation="%(organisation)s"
module="%(module)s"
revision="latest.integration"
conf="%(conf)s"
confs="%(conf)s"
date="%(timestamp)s"/>
</ivy-report>
"""
no_deps_xml = no_deps_xml_template % dict(
organisation=organisation,
module=module,
conf=conf,
timestamp=time.strftime('%Y%m%d%H%M%S'))
with open(report, 'w') as report_handle:
print(no_deps_xml, file=report_handle)
classpath = self._bootstrap_utils.get_jvm_build_tools_classpath(
self._ivy_bootstrap_key, self.runjava_indivisible)
reports = []
org, name = self._ivy_utils.identify(targets)
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
safe_mkdir(self._outdir, clean=True)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = self._ivy_utils.xml_report_path(targets, conf)
if not os.path.exists(xml):
make_empty_report(xml, org, name, conf)
out = os.path.join(self._outdir,
'%(org)s-%(name)s-%(conf)s.html' % params)
opts = ['-IN', xml, '-XSL', xsl, '-OUT', out]
if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process',
classpath=classpath,
opts=opts,
workunit_name='report'):
raise TaskError
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'),
self._outdir)
if self._open:
binary_util.ui_open(*reports)
