def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_bootstrap_key = 'ivy' ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan') self._bootstrap_utils.register_jvm_build_tools(self._ivy_bootstrap_key, ivy_bootstrap_tools) self._ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. artifact_cache_spec = context.config.getlist('ivy-resolve', 'artifact_caches', default=[]) self.setup_artifact_cache(artifact_cache_spec)
def __init__(self, context): Task.__init__(self, context) self.ivy_utils = IvyUtils(context, context.config.get('ivy', 'cache_dir')) self.confs = context.config.getlist('ivy', 'confs') self.target_roots = context.target_roots self.transitive = context.options.provides_transitive self.workdir = context.config.get('provides', 'workdir') self.outdir = context.options.provides_outdir or self.workdir self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False # Create a fake target, in case we were run directly on a JarLibrary containing nothing but JarDependencies. # TODO(benjy): Get rid of this special-casing of jar dependencies. context.add_new_target(self.workdir, JvmBinary, name='provides', dependencies=self.target_roots, configurations=self.confs) context.products.require('jars')
def __init__(self, context): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._args = context.config.getlist('ivy-resolve', 'args') self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url ) def replace_rev(template): context.log.info(fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info(fmt_message('Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides)
def __init__(self, context): classpath = context.config.getlist("ivy", "classpath") nailgun_dir = context.config.get("ivy-resolve", "nailgun_dir") NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get("ivy", "ivy_settings") self._cachedir = context.options.ivy_resolve_cache or context.config.get("ivy", "cache_dir") self._confs = context.config.getlist("ivy-resolve", "confs") self._transitive = context.config.getbool("ivy-resolve", "transitive") self._args = context.config.getlist("ivy-resolve", "args") self._profile = context.config.get("ivy-resolve", "profile") self._template_path = os.path.join("ivy_resolve", "ivy.mustache") self._work_dir = context.config.get("ivy-resolve", "workdir") self._classpath_file = os.path.join(self._work_dir, "classpath") self._classpath_dir = os.path.join(self._work_dir, "mapped") self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, "reports") self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r"^([^#]+)#([^=]+)=([^\s]+)$", override) if not match: raise TaskError("Invalid dependency override: %s" % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden="%s#%s;%s" % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url, ) def replace_rev(template): context.log.info(fmt_message("Overrode %(overridden)s with rev %(rev)s", template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info(fmt_message("Overrode %(overridden)s with snapshot at %(url)s", template)) return template.extend(version="SNAPSHOT", url=rev_or_url, force=True) replace = replace_url if re.match(r"^\w+://.+", rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides)
def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._cachedir = context.options.ivy_resolve_cache or context.config.get( 'ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_bootstrap_key = 'ivy' ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan') self._bootstrap_utils.register_jvm_build_tools(self._ivy_bootstrap_key, ivy_bootstrap_tools) self._ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. artifact_cache_spec = context.config.getlist('ivy-resolve', 'artifact_caches', default=[]) self.setup_artifact_cache(artifact_cache_spec)
class Provides(Task): @classmethod def setup_parser(cls, option_group, args, mkflag): option_group.add_option(mkflag("outdir"), dest="provides_outdir", help="Emit provides outputs into this directory.") option_group.add_option(mkflag("transitive"), default=False, action="store_true", dest='provides_transitive', help="Shows the symbols provided not just by the specified targets but by all their transitive dependencies.") option_group.add_option(mkflag("also-write-to-stdout"), default=False, action="store_true", dest='provides_also_write_to_stdout', help="If set, also outputs the provides information to stdout.") def __init__(self, context): Task.__init__(self, context) self.ivy_utils = IvyUtils(context, context.config.get('ivy', 'cache_dir')) self.confs = context.config.getlist('ivy', 'confs') self.target_roots = context.target_roots self.transitive = context.options.provides_transitive self.workdir = context.config.get('provides', 'workdir') self.outdir = context.options.provides_outdir or self.workdir self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False # Create a fake target, in case we were run directly on a JarLibrary containing nothing but JarDependencies. # TODO(benjy): Get rid of this special-casing of jar dependencies. context.add_new_target(self.workdir, JvmBinary, name='provides', dependencies=self.target_roots, configurations=self.confs) context.products.require('jars') def execute(self, targets): for conf in self.confs: outpath = os.path.join(self.outdir, '%s.%s.provides' % (self.ivy_utils.identify()[1], conf)) if self.transitive: outpath += '.transitive' ivyinfo = self.ivy_utils.parse_xml_report(conf) jar_paths = OrderedSet() for root in self.target_roots: jar_paths.update(self.get_jar_paths(ivyinfo, root, conf)) with open(outpath, 'w') as outfile: def do_write(s): outfile.write(s) if self.also_write_to_stdout: sys.stdout.write(s) for jar in jar_paths: do_write('# from jar %s\n' % jar) for line in self.list_jar(jar): if line.endswith('.class'): class_name = line[:-6].replace('/', '.') do_write(class_name) do_write('\n') print 'Wrote provides information to %s' % outpath def get_jar_paths(self, ivyinfo, target, conf): jar_paths = OrderedSet() if is_jar_library(target): # Jar library proxies jar dependencies or jvm targets, so the jars are just those of the # dependencies. for paths in [ self.get_jar_paths(ivyinfo, dep, conf) for dep in target.dependencies ]: jar_paths.update(paths) elif is_jar_dependency(target): ref = IvyModuleRef(target.org, target.name, target.rev, conf) jar_paths.update(self.get_jar_paths_for_ivy_module(ivyinfo, ref)) elif is_jvm(target): for basedir, jars in self.context.products.get('jars').get(target).items(): jar_paths.update([os.path.join(basedir, jar) for jar in jars]) if self.transitive: for dep in target.dependencies: jar_paths.update(self.get_jar_paths(ivyinfo, dep, conf)) return jar_paths def get_jar_paths_for_ivy_module(self, ivyinfo, ref): jar_paths = OrderedSet() module = ivyinfo.modules_by_ref[ref] jar_paths.update([a.path for a in module.artifacts]) if self.transitive: for dep in ivyinfo.deps_by_caller.get(ref, []): jar_paths.update(self.get_jar_paths_for_ivy_module(ivyinfo, dep)) return jar_paths def list_jar(self, path): with open_jar(path, 'r') as jar: return jar.namelist()
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option(flag, action='append', dest='ivy_resolve_overrides', help='''Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar ''' % dict(flag=flag)) report = mkflag("report") option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help = "[%default] Generate an ivy resolve html report") option_group.add_option(mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") option_group.add_option(mkflag("cache"), dest="ivy_resolve_cache", help="Use this directory as the ivy cache, instead of the " \ "default specified in pants.ini.") def __init__(self, context): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._args = context.config.getlist('ivy-resolve', 'args') self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url ) def replace_rev(template): context.log.info(fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info(fmt_message('Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides) def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """ Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ def dirname_for_requested_targets(targets): """Where we put the classpath file for this set of targets.""" sha = hashlib.sha1() for t in targets: sha.update(t.id) return sha.hexdigest() def is_classpath(t): return is_internal(t) and any(jar for jar in t.jar_dependencies if jar.rev) classpath_targets = filter(is_classpath, targets) target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(targets)) target_classpath_file = os.path.join(target_workdir, 'classpath') with self.invalidated(classpath_targets, only_buildfiles=True, invalidate_dependants=True) as invalidation_check: # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously build a superset of targets. if len(invalidation_check.invalid_vts) > 0 or not os.path.exists(target_classpath_file): self._exec_ivy(target_workdir, targets, [ '-cachepath', target_classpath_file, '-confs' ] + self._confs) if not os.path.exists(target_classpath_file): raise TaskError, 'Ivy failed to create classpath file at %s' % target_classpath_file def safe_link(src, dest): if os.path.exists(dest): os.unlink(dest) os.symlink(src, dest) # Symlink to the current classpath file. safe_link(target_classpath_file, self._classpath_file) # Symlink to the current ivy.xml file (useful for IDEs that read it). ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml') target_ivyxml = os.path.join(target_workdir, 'ivy.xml') safe_link(target_ivyxml, ivyxml_symlink) with self._cachepath(self._classpath_file) as classpath: with self.context.state('classpath', []) as cp: for path in classpath: if self._is_jar(path): for conf in self._confs: cp.append((conf, path.strip())) if self._report: self._generate_ivy_report() create_jardeps_for = self.context.products.isrequired('jar_dependencies') if create_jardeps_for: genmap = self.context.products.get('jar_dependencies') for target in filter(create_jardeps_for, targets): self._mapjars(genmap, target) def _generate_ivy(self, jars, excludes, ivyxml): org, name = self._ivy_utils.identify() template_data = TemplateData( org=org, module=name, version='latest.integration', publications=None, dependencies=[self._generate_jar_template(jar) for jar in jars], excludes=[self._generate_exclude_template(exclude) for exclude in excludes] ) safe_mkdir(os.path.dirname(ivyxml)) with open(ivyxml, 'w') as output: generator = Generator(pkgutil.get_data(__name__, self._template_path), root_dir = get_buildroot(), lib = template_data) generator.write(output) def _generate_ivy_report(self): classpath = binary_utils.nailgun_profile_classpath(self, self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict( org=org, name=name, conf=conf ) xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args) reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_utils.open(*reports) def _calculate_classpath(self, targets): jars = set() excludes = set() def collect_jars(target): if target.jar_dependencies: jars.update(jar for jar in target.jar_dependencies if jar.rev) if target.excludes: excludes.update(target.excludes) for target in targets: target.walk(collect_jars, is_jvm) return jars, excludes def _generate_jar_template(self, jar): template = TemplateData( org = jar.org, module = jar.name, version = jar.rev, force = jar.force, excludes = [self._generate_exclude_template(exclude) for exclude in jar.excludes], transitive = jar.transitive, artifacts = jar.artifacts, configurations = ';'.join(jar._configurations), ) override = self._overrides.get((jar.org, jar.name)) return override(template) if override else template def _generate_exclude_template(self, exclude): return TemplateData(org = exclude.org, name = exclude.name) @contextmanager def _cachepath(self, file): with safe_open(file, 'r') as cp: yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip()) def _mapjars(self, genmap, target): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self._classpath_dir, target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', '-confs', ] ivyargs.extend(target.configurations or self._confs) self._exec_ivy(mapdir, [target], ivyargs) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for file in os.listdir(confdir): if self._is_jar(file): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(file) genmap.add((org, name), confdir).append(file) genmap.add(target, confdir).append(file) genmap.add((target, conf), confdir).append(file) genmap.add((org, name, conf), confdir).append(file) def _is_jar(self, path): return path.endswith('.jar') def _exec_ivy(self, target_workdir, targets, args): ivyxml = os.path.join(target_workdir, 'ivy.xml') jars, excludes = self._calculate_classpath(targets) self._generate_ivy(jars, excludes, ivyxml) ivy_args = [ '-settings', self._ivy_settings, '-cache', self._cachedir, '-ivy', ivyxml, ] ivy_args.extend(args) if not self._transitive: ivy_args.append('-notransitive') ivy_args.extend(self._args) result = self.runjava('org.apache.ivy.Main', args=ivy_args) if result != 0: raise TaskError('org.apache.ivy.Main returned %d' % result)
def ivy_resolve(self, targets, java_runner=None, symlink_ivyxml=False, silent=False, workunit_name=None, workunit_labels=None): java_runner = java_runner or runjava_indivisible targets = set(targets) if not targets: return [] work_dir = self.context.config.get('ivy-resolve', 'workdir') confs = self.context.config.getlist('ivy-resolve', 'confs') with self.invalidated(targets, only_buildfiles=True, invalidate_dependents=True, silent=silent) as invalidation_check: global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) target_workdir = os.path.join(work_dir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' symlink_dir = os.path.join(target_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) args = (['-cachepath', raw_target_classpath_file_tmp] + ['-confs'] + confs) def exec_ivy(): ivy_utils.exec_ivy( target_workdir=target_workdir, targets=targets, args=args, runjava=java_runner, workunit_name='ivy', workunit_factory=self.context.new_workunit, symlink_ivyxml=symlink_ivyxml, ) if workunit_name: with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []): exec_ivy() else: exec_ivy() if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. symlink_map = IvyUtils.symlink_cachepath(raw_target_classpath_file, symlink_dir, target_classpath_file) with Task.symlink_map_lock: all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list) for path, symlink in symlink_map.items(): all_symlinks_map[os.path.realpath(path)].append(symlink) self.context.products.set_data('symlink_map', all_symlinks_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return [path for path in stripped_classpath if IvyUtils.is_mappable_artifact(path)]
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option(flag, action='append', dest='ivy_resolve_overrides', help="""Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar """ % dict(flag=flag)) report = mkflag("report") option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help = "[%default] Generate an ivy resolve html report") option_group.add_option(mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") option_group.add_option(mkflag("cache"), dest="ivy_resolve_cache", help="Use this directory as the ivy cache, instead of the " "default specified in pants.ini.") option_group.add_option(mkflag("mutable-pattern"), dest="ivy_mutable_pattern", help="If specified, all artifact revisions matching this pattern will " "be treated as mutable unless a matching artifact explicitly " "marks mutable as False.") def __init__(self, context, confs=None): nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, workdir=nailgun_dir) self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_bootstrap_key = 'ivy' ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan') self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools) self._ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache_from_config(config_section='ivy-resolve') def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ groups = self.context.products.get_data('exclusives_groups') # Below, need to take the code that actually execs ivy, and invoke it once for each # group. Then after running ivy, we need to take the resulting classpath, and load it into # the build products. # The set of groups we need to consider is complicated: # - If there are no conflicting exclusives (ie, there's only one entry in the map), # then we just do the one. # - If there are conflicts, then there will be at least three entries in the groups map: # - the group with no exclusives (X) # - the two groups that are in conflict (A and B). # In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X, # because things in A and B can depend on things in X; and so they can indirectly depend # on the dependencies of X. # (I think this well be covered by the computed transitive dependencies of # A and B. But before pushing this change, review this comment, and make sure that this is # working correctly.) for group_key in groups.get_group_keys(): # Narrow the groups target set to just the set of targets that we're supposed to build. # Normally, this shouldn't be different from the contents of the group. group_targets = groups.get_targets_for_group_key(group_key) & set(targets) # NOTE(pl): The symlinked ivy.xml (for IDEs, particularly IntelliJ) in the presence of # multiple exclusives groups will end up as the last exclusives group run. I'd like to # deprecate this eventually, but some people rely on it, and it's not clear to me right now # whether telling them to use IdeaGen instead is feasible. classpath = self.ivy_resolve(group_targets, java_runner=self.runjava_indivisible, symlink_ivyxml=True) if self.context.products.isrequired('ivy_jar_products'): self._populate_ivy_jar_products(group_targets) for conf in self._confs: for path in classpath: groups.update_compatible_classpaths(group_key, [(conf, path)]) if self._report: self._generate_ivy_report(group_targets) create_jardeps_for = self.context.products.isrequired(self._ivy_utils._mapfor_typename()) if create_jardeps_for: genmap = self.context.products.get(self._ivy_utils._mapfor_typename()) for target in filter(create_jardeps_for, targets): self._ivy_utils.mapjars(genmap, target, java_runner=self.runjava_indivisible) def check_artifact_cache_for(self, invalidation_check): # Ivy resolution is an output dependent on the entire target set, and is not divisible # by target. So we can only cache it keyed by the entire target set. global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) return [global_vts] def _populate_ivy_jar_products(self, targets): """Populate the build products with an IvyInfo object for each generated ivy report.""" ivy_products = self.context.products.get_data('ivy_jar_products') or defaultdict(list) for conf in self._confs: ivyinfo = self._ivy_utils.parse_xml_report(targets, conf) if ivyinfo: ivy_products[conf].append(ivyinfo) # Value is a list, to accommodate multiple exclusives groups. self.context.products.set_data('ivy_jar_products', ivy_products) def _generate_ivy_report(self, targets): def make_empty_report(report, organisation, module, conf): no_deps_xml_template = """ <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?> <ivy-report version="1.0"> <info organisation="%(organisation)s" module="%(module)s" revision="latest.integration" conf="%(conf)s" confs="%(conf)s" date="%(timestamp)s"/> </ivy-report> """ no_deps_xml = no_deps_xml_template % dict(organisation=organisation, module=module, conf=conf, timestamp=time.strftime('%Y%m%d%H%M%S')) with open(report, 'w') as report_handle: print(no_deps_xml, file=report_handle) classpath = self._jvm_tool_bootstrapper.get_jvm_tool_classpath(self._ivy_bootstrap_key, self.runjava_indivisible) reports = [] org, name = self._ivy_utils.identify(targets) xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = self._ivy_utils.xml_report_path(targets, conf) if not os.path.exists(xml): make_empty_report(xml, org, name, conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process', classpath=classpath, args=args, workunit_name='report'): raise TaskError reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_util.ui_open(*reports)
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option(flag, action='append', dest='ivy_resolve_overrides', help='''Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar ''' % dict(flag=flag)) report = mkflag("report") option_group.add_option(report, mkflag("report", negate=True), dest = "ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help = "[%default] Generate an ivy resolve html report") option_group.add_option(mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option(mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") option_group.add_option(mkflag("cache"), dest="ivy_resolve_cache", help="Use this directory as the ivy cache, instead of the " "default specified in pants.ini.") option_group.add_option(mkflag("args"), dest="ivy_args", action="append", default=[], help = "Pass these extra args to ivy.") option_group.add_option(mkflag("mutable-pattern"), dest="ivy_mutable_pattern", help="If specified, all artifact revisions matching this pattern will " "be treated as mutable unless a matching artifact explicitly " "marks mutable as False.") def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._opts = context.config.getlist('ivy-resolve', 'args') self._ivy_args = context.options.ivy_args self._mutable_pattern = (context.options.ivy_mutable_pattern or context.config.get('ivy-resolve', 'mutable_pattern', default=None)) if self._mutable_pattern: try: self._mutable_pattern = re.compile(self._mutable_pattern) except re.error as e: raise TaskError('Invalid mutable pattern specified: %s %s' % (self._mutable_pattern, e)) self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('templates', 'ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) context.products.require_data('exclusives_groups') def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url ) def replace_rev(template): context.log.info(fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info(fmt_message('Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides) def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ def dirname_for_requested_targets(targets): """Where we put the classpath file for this set of targets.""" sha = hashlib.sha1() for t in targets: sha.update(t.id) return sha.hexdigest() def is_classpath(target): return is_jar(target) or ( is_internal(target) and any(jar for jar in target.jar_dependencies if jar.rev) ) groups = self.context.products.get_data('exclusives_groups') # Below, need to take the code that actually execs ivy, and invoke it once for each # group. Then after running ivy, we need to take the resulting classpath, and load it into # the build products. # The set of groups we need to consider is complicated: # - If there are no conflicting exclusives (ie, there's only one entry in the map), # then we just do the one. # - If there are conflicts, then there will be at least three entries in the groups map: # - the group with no exclusives (X) # - the two groups that are in conflict (A and B). # In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X, # because things in A and B can depend on things in X; and so they can indirectly depend # on the dependencies of X. (I think this well be covered by the computed transitive dependencies of # A and B. But before pushing this change, review this comment, and make sure that this is # working correctly. for group_key in groups.get_group_keys(): # Narrow the groups target set to just the set of targets that we're supposed to build. # Normally, this shouldn't be different from the contents of the group. group_targets = groups.get_targets_for_group_key(group_key) & set(targets) classpath_targets = OrderedSet() for target in group_targets: classpath_targets.update(filter(is_classpath, filter(is_concrete, target.resolve()))) target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(group_targets)) target_classpath_file = os.path.join(target_workdir, 'classpath') with self.invalidated(classpath_targets, only_buildfiles=True, invalidate_dependents=True) as invalidation_check: # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously build a superset of targets. if invalidation_check.invalid_vts or not os.path.exists(target_classpath_file): self._exec_ivy(target_workdir, targets, [ '-cachepath', target_classpath_file, '-confs' ] + self._confs) if not os.path.exists(target_classpath_file): print ('Ivy failed to create classpath file at %s %s' % target_classpath_file) def safe_link(src, dest): if os.path.exists(dest): os.unlink(dest) os.symlink(src, dest) # TODO(benjy): Is this symlinking valid in the presence of multiple exclusives groups? # Should probably get rid of it and use a local artifact cache instead. # Symlink to the current classpath file. safe_link(target_classpath_file, self._classpath_file) # Symlink to the current ivy.xml file (useful for IDEs that read it). ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml') target_ivyxml = os.path.join(target_workdir, 'ivy.xml') safe_link(target_ivyxml, ivyxml_symlink) if os.path.exists(self._classpath_file): with self._cachepath(self._classpath_file) as classpath: for path in classpath: if self._map_jar(path): for conf in self._confs: groups.update_compatible_classpaths(group_key, [(conf, path.strip())]) if self._report: self._generate_ivy_report() if self.context.products.isrequired("ivy_jar_products"): self._populate_ivy_jar_products() create_jardeps_for = self.context.products.isrequired(self._mapfor_typename()) if create_jardeps_for: genmap = self.context.products.get(self._mapfor_typename()) for target in filter(create_jardeps_for, targets): self._mapjars(genmap, target) def _extract_classpathdeps(self, targets): """Subclasses can override to filter out a set of targets that should be resolved for classpath dependencies. """ def is_classpath(target): return is_jar(target) or ( is_internal(target) and any(jar for jar in target.jar_dependencies if jar.rev) ) classpath_deps = OrderedSet() for target in targets: classpath_deps.update(filter(is_classpath, filter(is_concrete, target.resolve()))) return classpath_deps def _generate_ivy(self, jars, excludes, ivyxml): org, name = self._ivy_utils.identify() template_data = TemplateData( org=org, module=name, version='latest.integration', publications=None, is_idl=False, dependencies=[self._generate_jar_template(jar) for jar in jars], excludes=[self._generate_exclude_template(exclude) for exclude in excludes] ) safe_mkdir(os.path.dirname(ivyxml)) with open(ivyxml, 'w') as output: generator = Generator(pkgutil.get_data(__name__, self._template_path), root_dir = get_buildroot(), lib = template_data) generator.write(output) def _populate_ivy_jar_products(self): """ Populate the build products with an IvyInfo object for each generated ivy report. For each configuration used to run ivy, a build product entry is generated for the tuple ("ivy", configuration, ivyinfo) """ genmap = self.context.products.get('ivy_jar_products') # For each of the ivy reports: for conf in self._confs: # parse the report file, and put it into the build products. # This is sort-of an abuse of the build-products. But build products # are already so abused, and this really does make sense. ivyinfo = self._ivy_utils.parse_xml_report(conf) genmap.add("ivy", conf, [ivyinfo]) def _generate_ivy_report(self): def make_empty_report(report, organisation, module, conf): no_deps_xml = """<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?> <ivy-report version="1.0"> <info organisation="%(organisation)s" module="%(module)s" revision="latest.integration" conf="%(conf)s" confs="%(conf)s" date="%(timestamp)s"/> </ivy-report>""" % dict(organisation=organisation, module=module, conf=conf, timestamp=time.strftime('%Y%m%d%H%M%S')) with open(report, 'w') as report_handle: print(no_deps_xml, file=report_handle) classpath = self.profile_classpath(self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict( org=org, name=name, conf=conf ) xml = os.path.join(self._cachedir, '%(org)s-%(name)s-%(conf)s.xml' % params) if not os.path.exists(xml): make_empty_report(xml, org, name, conf) #xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) opts = ['-IN', xml, '-XSL', xsl, '-OUT', out] if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process', classpath=classpath, opts=opts, workunit_name='report'): raise TaskError reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_util.ui_open(*reports) def _calculate_classpath(self, targets): def is_jardependant(target): return is_jar(target) or is_jvm(target) jars = {} excludes = set() # Support the ivy force concept when we sanely can for internal dep conflicts. # TODO(John Sirois): Consider supporting / implementing the configured ivy revision picking # strategy generally. def add_jar(jar): coordinate = (jar.org, jar.name) existing = jars.get(coordinate) jars[coordinate] = jar if not existing else ( self._resolve_conflict(existing=existing, proposed=jar) ) def collect_jars(target): if is_jar(target): add_jar(target) elif target.jar_dependencies: for jar in target.jar_dependencies: if jar.rev: add_jar(jar) # Lift jvm target-level excludes up to the global excludes set if is_jvm(target) and target.excludes: excludes.update(target.excludes) for target in targets: target.walk(collect_jars, is_jardependant) return jars.values(), excludes def _resolve_conflict(self, existing, proposed): if proposed == existing: return existing elif existing.force and proposed.force: raise TaskError('Cannot force %s#%s to both rev %s and %s' % ( proposed.org, proposed.name, existing.rev, proposed.rev )) elif existing.force: self.context.log.debug('Ignoring rev %s for %s#%s already forced to %s' % ( proposed.rev, proposed.org, proposed.name, existing.rev )) return existing elif proposed.force: self.context.log.debug('Forcing %s#%s from %s to %s' % ( proposed.org, proposed.name, existing.rev, proposed.rev )) return proposed else: try: if Revision.lenient(proposed.rev) > Revision.lenient(existing.rev): self.context.log.debug('Upgrading %s#%s from rev %s to %s' % ( proposed.org, proposed.name, existing.rev, proposed.rev, )) return proposed else: return existing except Revision.BadRevision as e: raise TaskError('Failed to parse jar revision', e) def _is_mutable(self, jar): if jar.mutable is not None: return jar.mutable if self._mutable_pattern: return self._mutable_pattern.match(jar.rev) return False def _generate_jar_template(self, jar): template=TemplateData( org=jar.org, module=jar.name, version=jar.rev, mutable=self._is_mutable(jar), force=jar.force, excludes=[self._generate_exclude_template(exclude) for exclude in jar.excludes], transitive=jar.transitive, artifacts=jar.artifacts, is_idl='idl' in jar._configurations, configurations=';'.join(jar._configurations), ) override = self._overrides.get((jar.org, jar.name)) return override(template) if override else template def _generate_exclude_template(self, exclude): return TemplateData(org=exclude.org, name=exclude.name) @contextmanager def _cachepath(self, path): if not os.path.exists(path): yield () else: with safe_open(path, 'r') as cp: yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip()) def _mapjars(self, genmap, target): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self._mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', '-confs', ] ivyargs.extend(target.configurations or self._confs) self._exec_ivy(mapdir, [target], ivyargs) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for file in os.listdir(confdir): if self._map_jar(file): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(file) genmap.add((org, name), confdir).append(file) genmap.add(target, confdir).append(file) genmap.add((target, conf), confdir).append(file) genmap.add((org, name, conf), confdir).append(file) def _mapfor_typename(self): """Subclasses can override to identify the product map typename that should trigger jar mapping. """ return 'jar_dependencies' def _mapto_dir(self): """Subclasses can override to establish an isolated jar mapping directory.""" return os.path.join(self._work_dir, 'mapped-jars') def _map_jar(self, path): """Subclasses can override to determine whether a given path represents a mappable artifact.""" return path.endswith('.jar') or path.endswith('.war') def _exec_ivy(self, target_workdir, targets, args): ivyxml = os.path.join(target_workdir, 'ivy.xml') jars, excludes = self._calculate_classpath(targets) self._generate_ivy(jars, excludes, ivyxml) ivy_opts = [ '-settings', self._ivy_settings, '-cache', self._cachedir, '-ivy', ivyxml, ] ivy_opts.extend(args) if not self._transitive: ivy_opts.append('-notransitive') ivy_opts.extend(self._opts) ivy_opts.extend(self._ivy_args) result = self.runjava_indivisible('org.apache.ivy.Main', opts=ivy_opts, workunit_name='ivy') if result != 0: raise TaskError('org.apache.ivy.Main returned %d' % result)
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option( flag, action='append', dest='ivy_resolve_overrides', help='''Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar ''' % dict(flag=flag)) report = mkflag("report") option_group.add_option( report, mkflag("report", negate=True), dest="ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help="[%default] Generate an ivy resolve html report") option_group.add_option( mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option( mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") option_group.add_option( mkflag("cache"), dest="ivy_resolve_cache", help="Use this directory as the ivy cache, instead of the " "default specified in pants.ini.") option_group.add_option(mkflag("args"), dest="ivy_args", action="append", default=[], help="Pass these extra args to ivy.") option_group.add_option( mkflag("mutable-pattern"), dest="ivy_mutable_pattern", help= "If specified, all artifact revisions matching this pattern will " "be treated as mutable unless a matching artifact explicitly " "marks mutable as False.") def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._cachedir = context.options.ivy_resolve_cache or context.config.get( 'ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_bootstrap_key = 'ivy' ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan') self._bootstrap_utils.register_jvm_build_tools(self._ivy_bootstrap_key, ivy_bootstrap_tools) self._ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. artifact_cache_spec = context.config.getlist('ivy-resolve', 'artifact_caches', default=[]) self.setup_artifact_cache(artifact_cache_spec) def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ groups = self.context.products.get_data('exclusives_groups') # Below, need to take the code that actually execs ivy, and invoke it once for each # group. Then after running ivy, we need to take the resulting classpath, and load it into # the build products. # The set of groups we need to consider is complicated: # - If there are no conflicting exclusives (ie, there's only one entry in the map), # then we just do the one. # - If there are conflicts, then there will be at least three entries in the groups map: # - the group with no exclusives (X) # - the two groups that are in conflict (A and B). # In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X, # because things in A and B can depend on things in X; and so they can indirectly depend # on the dependencies of X. # (I think this well be covered by the computed transitive dependencies of # A and B. But before pushing this change, review this comment, and make sure that this is # working correctly.) for group_key in groups.get_group_keys(): # Narrow the groups target set to just the set of targets that we're supposed to build. # Normally, this shouldn't be different from the contents of the group. group_targets = groups.get_targets_for_group_key(group_key) & set( targets) # NOTE(pl): The symlinked ivy.xml (for IDEs, particularly IntelliJ) in the presence of # multiple exclusives groups will end up as the last exclusives group run. I'd like to # deprecate this eventually, but some people rely on it, and it's not clear to me right now # whether telling them to use IdeaGen instead is feasible. classpath = self.ivy_resolve(group_targets, java_runner=self.runjava_indivisible, symlink_ivyxml=True) for conf in self._confs: for path in classpath: groups.update_compatible_classpaths( group_key, [(conf, path)]) if self._report: self._generate_ivy_report(group_targets) if self.context.products.isrequired('ivy_jar_products'): self._populate_ivy_jar_products(targets) create_jardeps_for = self.context.products.isrequired( self._ivy_utils._mapfor_typename()) if create_jardeps_for: genmap = self.context.products.get( self._ivy_utils._mapfor_typename()) for target in filter(create_jardeps_for, targets): self._ivy_utils.mapjars(genmap, target, java_runner=self.runjava_indivisible) def check_artifact_cache_for(self, invalidation_check): # Ivy resolution is an output dependent on the entire target set, and is not divisible # by target. So we can only cache it keyed by the entire target set. global_vts = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts) return [global_vts] def _populate_ivy_jar_products(self, targets): """Populate the build products with an IvyInfo object for each generated ivy report.""" ivy_products = {} for conf in self._confs: ivyinfo = self._ivy_utils.parse_xml_report(targets, conf) if ivyinfo: ivy_products[conf] = ivyinfo self.context.products.set_data('ivy_jar_products', ivy_products) def _generate_ivy_report(self, targets): def make_empty_report(report, organisation, module, conf): no_deps_xml_template = """ <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?> <ivy-report version="1.0"> <info organisation="%(organisation)s" module="%(module)s" revision="latest.integration" conf="%(conf)s" confs="%(conf)s" date="%(timestamp)s"/> </ivy-report> """ no_deps_xml = no_deps_xml_template % dict( organisation=organisation, module=module, conf=conf, timestamp=time.strftime('%Y%m%d%H%M%S')) with open(report, 'w') as report_handle: print(no_deps_xml, file=report_handle) classpath = self._bootstrap_utils.get_jvm_build_tools_classpath( self._ivy_bootstrap_key, self.runjava_indivisible) reports = [] org, name = self._ivy_utils.identify(targets) xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = self._ivy_utils.xml_report_path(targets, conf) if not os.path.exists(xml): make_empty_report(xml, org, name, conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) opts = ['-IN', xml, '-XSL', xsl, '-OUT', out] if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process', classpath=classpath, opts=opts, workunit_name='report'): raise TaskError reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_util.ui_open(*reports)
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option( flag, action='append', dest='ivy_resolve_overrides', help='''Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar ''' % dict(flag=flag)) report = mkflag("report") option_group.add_option( report, mkflag("report", negate=True), dest="ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help="[%default] Generate an ivy resolve html report") option_group.add_option( mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option( mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") def __init__(self, context): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.config.get('ivy', 'cache_dir') self._confs = context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._args = context.config.getlist('ivy-resolve', 'args') self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('ivy_resolve', 'ivy.mk') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url) def replace_rev(template): context.log.info( fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info( fmt_message( 'Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update( parse_override(o) for o in context.options.ivy_resolve_overrides) def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """ Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ def dirname_for_requested_targets(targets): """Where we put the classpath file for this set of targets.""" sha = hashlib.sha1() for t in targets: sha.update(t.id) return sha.hexdigest() def is_classpath(t): return is_internal(t) and any( jar for jar in t.jar_dependencies if jar.rev) target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(targets)) target_classpath_file = os.path.join(target_workdir, 'classpath') with self.changed(filter(is_classpath, targets), only_buildfiles=True) as changed_deps: if changed_deps: self._exec_ivy( target_workdir, targets, ['-cachepath', target_classpath_file, '-confs'] + self._confs) if os.path.exists(target_classpath_file): def safe_link(src, dest): if os.path.exists(dest): os.unlink(dest) os.symlink(src, dest) # Symlink to the current classpath file. safe_link(target_classpath_file, self._classpath_file) # Symlink to the current ivy.xml file (useful for IDEs that read it). ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml') target_ivyxml = os.path.join(target_workdir, 'ivy.xml') safe_link(target_ivyxml, ivyxml_symlink) if os.path.exists(self._classpath_file): with self._cachepath(self._classpath_file) as classpath: with self.context.state('classpath', []) as cp: for path in classpath: if self._is_jar(path): for conf in self._confs: cp.append((conf, path.strip())) if self._report: self._generate_ivy_report() create_jardeps_for = self.context.products.isrequired( 'jar_dependencies') if create_jardeps_for: genmap = self.context.products.get('jar_dependencies') for target in filter(create_jardeps_for, targets): self._mapjars(genmap, target) def _generate_ivy(self, jars, excludes, ivyxml): org, name = self._ivy_utils.identify() template_data = TemplateData( org=org, module=name, version='latest.integration', publications=None, dependencies=[self._generate_jar_template(jar) for jar in jars], excludes=[ self._generate_exclude_template(exclude) for exclude in excludes ]) safe_mkdir(os.path.dirname(ivyxml)) with open(ivyxml, 'w') as output: generator = Generator(pkgutil.get_data(__name__, self._template_path), root_dir=get_buildroot(), lib=template_data) generator.write(output) def _generate_ivy_report(self): classpath = binary_utils.nailgun_profile_classpath(self, self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] self.runjava('org.apache.xalan.xslt.Process', classpath=classpath, args=args) reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_utils.open(*reports) def _calculate_classpath(self, targets): jars = set() excludes = set() def collect_jars(target): if target.jar_dependencies: jars.update(jar for jar in target.jar_dependencies if jar.rev) if target.excludes: excludes.update(target.excludes) for target in targets: target.walk(collect_jars, is_jvm) return jars, excludes def _generate_jar_template(self, jar): template = TemplateData( org=jar.org, module=jar.name, version=jar.rev, force=jar.force, excludes=[ self._generate_exclude_template(exclude) for exclude in jar.excludes ], transitive=jar.transitive, ext=jar.ext, url=jar.url, configurations=';'.join(jar._configurations), ) override = self._overrides.get((jar.org, jar.name)) return override(template) if override else template def _generate_exclude_template(self, exclude): return TemplateData(org=exclude.org, name=exclude.name) @contextmanager def _cachepath(self, file): with safe_open(file, 'r') as cp: yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip()) def _mapjars(self, genmap, target): mapdir = os.path.join(self._classpath_dir, target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', '-confs', ] ivyargs.extend(target.configurations or self._confs) self._exec_ivy(mapdir, [target], ivyargs) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for file in os.listdir(confdir): if self._is_jar(file): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(file) genmap.add((org, name), confdir).append(file) genmap.add(target, confdir).append(file) genmap.add((target, conf), confdir).append(file) genmap.add((org, name, conf), confdir).append(file) def _is_jar(self, path): return path.endswith('.jar') def _exec_ivy(self, target_workdir, targets, args): ivyxml = os.path.join(target_workdir, 'ivy.xml') jars, excludes = self._calculate_classpath(targets) self._generate_ivy(jars, excludes, ivyxml) ivy_args = [ '-settings', self._ivy_settings, '-cache', self._cachedir, '-ivy', ivyxml, ] ivy_args.extend(args) if not self._transitive: ivy_args.append('-notransitive') ivy_args.extend(self._args) result = self.runjava('org.apache.ivy.Main', args=ivy_args) if result != 0: raise TaskError('org.apache.ivy.Main returned %d' % result)
class Provides(Task): @classmethod def setup_parser(cls, option_group, args, mkflag): option_group.add_option( mkflag("outdir"), dest="provides_outdir", help="Emit provides outputs into this directory.") option_group.add_option( mkflag("transitive"), default=False, action="store_true", dest='provides_transitive', help= "Shows the symbols provided not just by the specified targets but by all their transitive dependencies." ) option_group.add_option( mkflag("also-write-to-stdout"), default=False, action="store_true", dest='provides_also_write_to_stdout', help="If set, also outputs the provides information to stdout.") def __init__(self, context): Task.__init__(self, context) self.ivy_utils = IvyUtils(context, context.config.get('ivy', 'cache_dir')) self.confs = context.config.getlist('ivy', 'confs') self.target_roots = context.target_roots self.transitive = context.options.provides_transitive self.workdir = context.config.get('provides', 'workdir') self.outdir = context.options.provides_outdir or self.workdir self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False # Create a fake target, in case we were run directly on a JarLibrary containing nothing but JarDependencies. # TODO(benjy): Get rid of this special-casing of jar dependencies. context.add_new_target(self.workdir, JvmBinary, name='provides', dependencies=self.target_roots, configurations=self.confs) context.products.require('jars') def execute(self, targets): for conf in self.confs: outpath = os.path.join( self.outdir, '%s.%s.provides' % (self.ivy_utils.identify()[1], conf)) if self.transitive: outpath += '.transitive' ivyinfo = self.ivy_utils.parse_xml_report(conf) jar_paths = OrderedSet() for root in self.target_roots: jar_paths.update(self.get_jar_paths(ivyinfo, root, conf)) with open(outpath, 'w') as outfile: def do_write(s): outfile.write(s) if self.also_write_to_stdout: sys.stdout.write(s) for jar in jar_paths: do_write('# from jar %s\n' % jar) for line in self.list_jar(jar): if line.endswith('.class'): class_name = line[:-6].replace('/', '.') do_write(class_name) do_write('\n') print 'Wrote provides information to %s' % outpath def get_jar_paths(self, ivyinfo, target, conf): jar_paths = OrderedSet() if is_jar_library(target): # Jar library proxies jar dependencies or jvm targets, so the jars are just those of the # dependencies. for paths in [ self.get_jar_paths(ivyinfo, dep, conf) for dep in target.dependencies ]: jar_paths.update(paths) elif is_jar_dependency(target): ref = IvyModuleRef(target.org, target.name, target.rev, conf) jar_paths.update(self.get_jar_paths_for_ivy_module(ivyinfo, ref)) elif is_jvm(target): for basedir, jars in self.context.products.get('jars').get( target).items(): jar_paths.update([os.path.join(basedir, jar) for jar in jars]) if self.transitive: for dep in target.dependencies: jar_paths.update(self.get_jar_paths(ivyinfo, dep, conf)) return jar_paths def get_jar_paths_for_ivy_module(self, ivyinfo, ref): jar_paths = OrderedSet() module = ivyinfo.modules_by_ref[ref] jar_paths.update([a.path for a in module.artifacts]) if self.transitive: for dep in ivyinfo.deps_by_caller.get(ref, []): jar_paths.update( self.get_jar_paths_for_ivy_module(ivyinfo, dep)) return jar_paths def list_jar(self, path): with open_jar(path, 'r') as jar: return jar.namelist()
class IvyResolve(NailgunTask): @classmethod def setup_parser(cls, option_group, args, mkflag): NailgunTask.setup_parser(option_group, args, mkflag) flag = mkflag('override') option_group.add_option( flag, action='append', dest='ivy_resolve_overrides', help='''Specifies a jar dependency override in the form: [org]#[name]=(revision|url) For example, to specify 2 overrides: %(flag)s=com.foo#bar=0.1.2 \\ %(flag)s=com.baz#spam=file:///tmp/spam.jar ''' % dict(flag=flag)) report = mkflag("report") option_group.add_option( report, mkflag("report", negate=True), dest="ivy_resolve_report", action="callback", callback=mkflag.set_bool, default=False, help="[%default] Generate an ivy resolve html report") option_group.add_option( mkflag("open"), mkflag("open", negate=True), dest="ivy_resolve_open", default=False, action="callback", callback=mkflag.set_bool, help="[%%default] Attempt to open the generated ivy resolve report " "in a browser (implies %s)." % report) option_group.add_option( mkflag("outdir"), dest="ivy_resolve_outdir", help="Emit ivy report outputs in to this directory.") option_group.add_option( mkflag("cache"), dest="ivy_resolve_cache", help="Use this directory as the ivy cache, instead of the " "default specified in pants.ini.") option_group.add_option(mkflag("args"), dest="ivy_args", action="append", default=[], help="Pass these extra args to ivy.") option_group.add_option( mkflag("mutable-pattern"), dest="ivy_mutable_pattern", help= "If specified, all artifact revisions matching this pattern will " "be treated as mutable unless a matching artifact explicitly " "marks mutable as False.") def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get( 'ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._opts = context.config.getlist('ivy-resolve', 'args') self._ivy_args = context.options.ivy_args self._mutable_pattern = (context.options.ivy_mutable_pattern or context.config.get('ivy-resolve', 'mutable_pattern', default=None)) if self._mutable_pattern: try: self._mutable_pattern = re.compile(self._mutable_pattern) except re.error as e: raise TaskError('Invalid mutable pattern specified: %s %s' % (self._mutable_pattern, e)) self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('templates', 'ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url) def replace_rev(template): context.log.info( fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info( fmt_message( 'Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update( parse_override(o) for o in context.options.ivy_resolve_overrides) def invalidate_for(self): return self.context.options.ivy_resolve_overrides def execute(self, targets): """ Resolves the specified confs for the configured targets and returns an iterator over tuples of (conf, jar path). """ def dirname_for_requested_targets(targets): """Where we put the classpath file for this set of targets.""" sha = hashlib.sha1() for t in targets: sha.update(t.id) return sha.hexdigest() def is_classpath(target): return is_jar(target) or (is_internal(target) and any( jar for jar in target.jar_dependencies if jar.rev)) classpath_targets = OrderedSet() for target in targets: classpath_targets.update( filter(is_classpath, filter(is_concrete, target.resolve()))) target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(targets)) target_classpath_file = os.path.join(target_workdir, 'classpath') with self.invalidated( classpath_targets, only_buildfiles=True, invalidate_dependents=True) as invalidation_check: # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously build a superset of targets. if len(invalidation_check.invalid_vts) > 0 or not os.path.exists( target_classpath_file): self._exec_ivy( target_workdir, targets, ['-cachepath', target_classpath_file, '-confs'] + self._confs) if not os.path.exists(target_classpath_file): raise TaskError('Ivy failed to create classpath file at %s' % target_classpath_file) def safe_link(src, dest): if os.path.exists(dest): os.unlink(dest) os.symlink(src, dest) # Symlink to the current classpath file. safe_link(target_classpath_file, self._classpath_file) # Symlink to the current ivy.xml file (useful for IDEs that read it). ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml') target_ivyxml = os.path.join(target_workdir, 'ivy.xml') safe_link(target_ivyxml, ivyxml_symlink) if os.path.exists(self._classpath_file): with self._cachepath(self._classpath_file) as classpath: with self.context.state('classpath', []) as cp: for path in classpath: if self._map_jar(path): for conf in self._confs: cp.append((conf, path.strip())) if self._report: self._generate_ivy_report() if self.context.products.isrequired("ivy_jar_products"): self._populate_ivy_jar_products() create_jardeps_for = self.context.products.isrequired( self._mapfor_typename()) if create_jardeps_for: genmap = self.context.products.get(self._mapfor_typename()) for target in filter(create_jardeps_for, targets): self._mapjars(genmap, target) def _extract_classpathdeps(self, targets): """Subclasses can override to filter out a set of targets that should be resolved for classpath dependencies. """ def is_classpath(target): return is_jar(target) or (is_internal(target) and any( jar for jar in target.jar_dependencies if jar.rev)) classpath_deps = OrderedSet() for target in targets: classpath_deps.update( filter(is_classpath, filter(is_concrete, target.resolve()))) return classpath_deps def _generate_ivy(self, jars, excludes, ivyxml): org, name = self._ivy_utils.identify() template_data = TemplateData( org=org, module=name, version='latest.integration', publications=None, is_idl=False, dependencies=[self._generate_jar_template(jar) for jar in jars], excludes=[ self._generate_exclude_template(exclude) for exclude in excludes ]) safe_mkdir(os.path.dirname(ivyxml)) with open(ivyxml, 'w') as output: generator = Generator(pkgutil.get_data(__name__, self._template_path), root_dir=get_buildroot(), lib=template_data) generator.write(output) def _populate_ivy_jar_products(self): """ Populate the build products with an IvyInfo object for each generated ivy report. For each configuration used to run ivy, a build product entry is generated for the tuple ("ivy", configuration, ivyinfo) """ genmap = self.context.products.get('ivy_jar_products') # For each of the ivy reports: for conf in self._confs: # parse the report file, and put it into the build products. # This is sort-of an abuse of the build-products. But build products # are already so abused, and this really does make sense. ivyinfo = self._ivy_utils.parse_xml_report(conf) genmap.add("ivy", conf, [ivyinfo]) def _generate_ivy_report(self): def make_empty_report(report, organisation, module, conf): no_deps_xml = """<?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?> <ivy-report version="1.0"> <info organisation="%(organisation)s" module="%(module)s" revision="latest.integration" conf="%(conf)s" confs="%(conf)s" date="%(timestamp)s"/> </ivy-report>""" % dict(organisation=organisation, module=module, conf=conf, timestamp=time.strftime('%Y%m%d%H%M%S')) with open(report, 'w') as report_handle: print(no_deps_xml, file=report_handle) classpath = self.profile_classpath(self._profile) reports = [] org, name = self._ivy_utils.identify() xsl = os.path.join(self._cachedir, 'ivy-report.xsl') safe_mkdir(self._outdir, clean=True) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = os.path.join(self._cachedir, '%(org)s-%(name)s-%(conf)s.xml' % params) if not os.path.exists(xml): make_empty_report(xml, org, name, conf) #xml = self._ivy_utils.xml_report_path(conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) opts = ['-IN', xml, '-XSL', xsl, '-OUT', out] if 0 != self.runjava_indivisible('org.apache.xalan.xslt.Process', classpath=classpath, opts=opts, workunit_name='report'): raise TaskError reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_util.ui_open(*reports) def _calculate_classpath(self, targets): def is_jardependant(target): return is_jar(target) or is_jvm(target) jars = {} excludes = set() # Support the ivy force concept when we sanely can for internal dep conflicts. # TODO(John Sirois): Consider supporting / implementing the configured ivy revision picking # strategy generally. def add_jar(jar): coordinate = (jar.org, jar.name) existing = jars.get(coordinate) jars[coordinate] = jar if not existing else ( self._resolve_conflict(existing=existing, proposed=jar)) def collect_jars(target): if is_jar(target): add_jar(target) elif target.jar_dependencies: for jar in target.jar_dependencies: if jar.rev: add_jar(jar) # Lift jvm target-level excludes up to the global excludes set if is_jvm(target) and target.excludes: excludes.update(target.excludes) for target in targets: target.walk(collect_jars, is_jardependant) return jars.values(), excludes def _resolve_conflict(self, existing, proposed): if proposed == existing: return existing elif existing.force and proposed.force: raise TaskError( 'Cannot force %s#%s to both rev %s and %s' % (proposed.org, proposed.name, existing.rev, proposed.rev)) elif existing.force: self.context.log.debug( 'Ignoring rev %s for %s#%s already forced to %s' % (proposed.rev, proposed.org, proposed.name, existing.rev)) return existing elif proposed.force: self.context.log.debug( 'Forcing %s#%s from %s to %s' % (proposed.org, proposed.name, existing.rev, proposed.rev)) return proposed else: try: if Revision.lenient(proposed.rev) > Revision.lenient( existing.rev): self.context.log.debug( 'Upgrading %s#%s from rev %s to %s' % ( proposed.org, proposed.name, existing.rev, proposed.rev, )) return proposed else: return existing except Revision.BadRevision as e: raise TaskError('Failed to parse jar revision', e) def _is_mutable(self, jar): if jar.mutable is not None: return jar.mutable if self._mutable_pattern: return self._mutable_pattern.match(jar.rev) return False def _generate_jar_template(self, jar): template = TemplateData( org=jar.org, module=jar.name, version=jar.rev, mutable=self._is_mutable(jar), force=jar.force, excludes=[ self._generate_exclude_template(exclude) for exclude in jar.excludes ], transitive=jar.transitive, artifacts=jar.artifacts, is_idl='idl' in jar._configurations, configurations=';'.join(jar._configurations), ) override = self._overrides.get((jar.org, jar.name)) return override(template) if override else template def _generate_exclude_template(self, exclude): return TemplateData(org=exclude.org, name=exclude.name) @contextmanager def _cachepath(self, file): if not os.path.exists(file): yield () else: with safe_open(file, 'r') as cp: yield (path.strip() for path in cp.read().split(os.pathsep) if path.strip()) def _mapjars(self, genmap, target): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self._mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', '-confs', ] ivyargs.extend(target.configurations or self._confs) self._exec_ivy(mapdir, [target], ivyargs) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for file in os.listdir(confdir): if self._map_jar(file): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(file) genmap.add((org, name), confdir).append(file) genmap.add(target, confdir).append(file) genmap.add((target, conf), confdir).append(file) genmap.add((org, name, conf), confdir).append(file) def _mapfor_typename(self): """Subclasses can override to identify the product map typename that should trigger jar mapping. """ return 'jar_dependencies' def _mapto_dir(self): """Subclasses can override to establish an isolated jar mapping directory.""" return os.path.join(self._work_dir, 'mapped-jars') def _map_jar(self, path): """Subclasses can override to determine whether a given path represents a mappable artifact.""" return path.endswith('.jar') def _exec_ivy(self, target_workdir, targets, args): ivyxml = os.path.join(target_workdir, 'ivy.xml') jars, excludes = self._calculate_classpath(targets) self._generate_ivy(jars, excludes, ivyxml) ivy_opts = [ '-settings', self._ivy_settings, '-cache', self._cachedir, '-ivy', ivyxml, ] ivy_opts.extend(args) if not self._transitive: ivy_opts.append('-notransitive') ivy_opts.extend(self._opts) ivy_opts.extend(self._ivy_args) result = self.runjava_indivisible('org.apache.ivy.Main', opts=ivy_opts, workunit_name='ivy') if result != 0: raise TaskError('org.apache.ivy.Main returned %d' % result)
def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get( 'ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._opts = context.config.getlist('ivy-resolve', 'args') self._ivy_args = context.options.ivy_args self._mutable_pattern = (context.options.ivy_mutable_pattern or context.config.get('ivy-resolve', 'mutable_pattern', default=None)) if self._mutable_pattern: try: self._mutable_pattern = re.compile(self._mutable_pattern) except re.error as e: raise TaskError('Invalid mutable pattern specified: %s %s' % (self._mutable_pattern, e)) self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('templates', 'ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. artifact_cache_spec = context.config.getlist('ivy-resolve', 'artifact_caches2', default=[]) self.setup_artifact_cache(artifact_cache_spec) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url) def replace_rev(template): context.log.info( fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info( fmt_message( 'Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update( parse_override(o) for o in context.options.ivy_resolve_overrides)
def ivy_resolve(self, targets, java_runner=None, ivy_args=None, symlink_ivyxml=False, silent=False, workunit_name=None, workunit_labels=None): java_runner = java_runner or runjava_indivisible ivy_args = ivy_args or [] targets = set(targets) if not targets: return [] work_dir = self.context.config.get('ivy-resolve', 'workdir') confs = self.context.config.getlist('ivy-resolve', 'confs') with self.invalidated(targets, only_buildfiles=True, invalidate_dependents=True, silent=silent) as invalidation_check: global_vts = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts) target_workdir = os.path.join(work_dir, global_vts.cache_key.hash) target_classpath_file = os.path.join(target_workdir, 'classpath') target_classpath_file_tmp = target_classpath_file + '.tmp' # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if invalidation_check.invalid_vts or not os.path.exists( target_classpath_file): ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) args = (['-cachepath', target_classpath_file_tmp] + ['-confs'] + confs + ivy_args) def exec_ivy(): ivy_utils.exec_ivy( target_workdir=target_workdir, targets=targets, args=args, runjava=java_runner, workunit_name='ivy', workunit_factory=self.context.new_workunit, symlink_ivyxml=symlink_ivyxml, ) if workunit_name: with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []): exec_ivy() else: exec_ivy() if not os.path.exists(target_classpath_file_tmp): raise TaskError( 'Ivy failed to create classpath file at %s' % target_classpath_file_tmp) shutil.move(target_classpath_file_tmp, target_classpath_file) if self.get_artifact_cache( ) and self.context.options.write_to_artifact_cache: self.update_artifact_cache([(global_vts, [target_classpath_file])]) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return [ path for path in stripped_classpath if IvyUtils.is_mappable_artifact(path) ]
def __init__(self, context): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.config.get('ivy', 'cache_dir') self._confs = context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._args = context.config.getlist('ivy-resolve', 'args') self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('ivy_resolve', 'ivy.mk') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_file = os.path.join(self._work_dir, 'classpath') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url) def replace_rev(template): context.log.info( fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info( fmt_message( 'Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update( parse_override(o) for o in context.options.ivy_resolve_overrides)
def __init__(self, context, confs=None): classpath = context.config.getlist('ivy', 'classpath') nailgun_dir = context.config.get('ivy-resolve', 'nailgun_dir') NailgunTask.__init__(self, context, classpath=classpath, workdir=nailgun_dir) self._ivy_settings = context.config.get('ivy', 'ivy_settings') self._cachedir = context.options.ivy_resolve_cache or context.config.get('ivy', 'cache_dir') self._confs = confs or context.config.getlist('ivy-resolve', 'confs') self._transitive = context.config.getbool('ivy-resolve', 'transitive') self._opts = context.config.getlist('ivy-resolve', 'args') self._ivy_args = context.options.ivy_args self._mutable_pattern = (context.options.ivy_mutable_pattern or context.config.get('ivy-resolve', 'mutable_pattern', default=None)) if self._mutable_pattern: try: self._mutable_pattern = re.compile(self._mutable_pattern) except re.error as e: raise TaskError('Invalid mutable pattern specified: %s %s' % (self._mutable_pattern, e)) self._profile = context.config.get('ivy-resolve', 'profile') self._template_path = os.path.join('templates', 'ivy_resolve', 'ivy.mustache') self._work_dir = context.config.get('ivy-resolve', 'workdir') self._classpath_dir = os.path.join(self._work_dir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join(self._work_dir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_utils = IvyUtils(context, self._cachedir) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. artifact_cache_spec = context.config.getlist('ivy-resolve', 'artifact_caches2', default=[]) self.setup_artifact_cache(artifact_cache_spec) def parse_override(override): match = re.match(r'^([^#]+)#([^=]+)=([^\s]+)$', override) if not match: raise TaskError('Invalid dependency override: %s' % override) org, name, rev_or_url = match.groups() def fmt_message(message, template): return message % dict( overridden='%s#%s;%s' % (template.org, template.module, template.version), rev=rev_or_url, url=rev_or_url ) def replace_rev(template): context.log.info(fmt_message('Overrode %(overridden)s with rev %(rev)s', template)) return template.extend(version=rev_or_url, url=None, force=True) def replace_url(template): context.log.info(fmt_message('Overrode %(overridden)s with snapshot at %(url)s', template)) return template.extend(version='SNAPSHOT', url=rev_or_url, force=True) replace = replace_url if re.match(r'^\w+://.+', rev_or_url) else replace_rev return (org, name), replace self._overrides = {} if context.options.ivy_resolve_overrides: self._overrides.update(parse_override(o) for o in context.options.ivy_resolve_overrides)