def publish(ivyxml_path):
ivysettings = self.generate_ivysettings(published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.outdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.outdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name = 'jar-publish')
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def __init__(self, context, confs=None):
super(IvyResolve, self).__init__(context)
work_dir = context.config.get('ivy-resolve', 'workdir')
self._ivy_bootstrapper = Bootstrapper.instance()
self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
self._confs = confs or context.config.getlist(
'ivy-resolve', 'confs', default=['default'])
self._classpath_dir = os.path.join(work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(
work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve',
'bootstrap-tools',
':xalan')
self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key,
ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def publish(ivyxml_path):
ivysettings = self.generate_ivysettings(published, publish_local=path)
args = [
'-settings', ivysettings,
'-ivy', ivyxml_path,
'-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.outdir,
'-publish', resolver,
'-publishpattern', '%s/[organisation]/[module]/'
'[artifact]-[revision](-[classifier]).[ext]' % self.outdir,
'-revision', newver.version(),
'-m2compatible',
]
if LogOptions.stderr_log_level() == logging.DEBUG:
args.append('-verbose')
if self.snapshot:
args.append('-overwrite')
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(jvm_options=jvm_args, args=args,
workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
except (Bootstrapper.Error, Ivy.Error) as e:
raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def exec_ivy(self,
target_workdir,
targets,
args,
confs=None,
ivy=None,
workunit_name='ivy',
workunit_factory=None,
symlink_ivyxml=False):
ivy = ivy or Bootstrapper.default_ivy()
if not isinstance(ivy, Ivy):
raise ValueError(
'The ivy argument supplied must be an Ivy instance, given %s of type %s'
% (ivy, type(ivy)))
ivyxml = os.path.join(target_workdir, 'ivy.xml')
jars, excludes = self._calculate_classpath(targets)
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
def safe_link(src, dest):
if os.path.exists(dest):
os.unlink(dest)
os.symlink(src, dest)
with IvyUtils.ivy_lock:
self._generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve)
runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
try:
result = util.execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
if symlink_ivyxml:
ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
safe_link(ivyxml, ivyxml_symlink)
if result != 0:
raise TaskError('Ivy returned %d' % result)
except runner.executor.Error as e:
raise TaskError(e)
def exec_ivy(self,
target_workdir,
targets,
args,
confs=None,
ivy=None,
workunit_name='ivy',
workunit_factory=None,
symlink_ivyxml=False):
ivy = ivy or Bootstrapper.default_ivy()
if not isinstance(ivy, Ivy):
raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s'
% (ivy, type(ivy)))
ivyxml = os.path.join(target_workdir, 'ivy.xml')
jars, excludes = self._calculate_classpath(targets)
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
if not self._transitive:
ivy_args.append('-notransitive')
ivy_args.extend(self._args)
def safe_link(src, dest):
if os.path.exists(dest):
os.unlink(dest)
os.symlink(src, dest)
with IvyUtils.ivy_lock:
self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve)
runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
try:
result = util.execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name)
# Symlink to the current ivy.xml file (useful for IDEs that read it).
if symlink_ivyxml:
ivyxml_symlink = os.path.join(self._work_dir, 'ivy.xml')
safe_link(ivyxml, ivyxml_symlink)
if result != 0:
raise TaskError('Ivy returned %d' % result)
except runner.executor.Error as e:
raise TaskError(e)
def run_antlrs(self, output_dir):
args = [
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar',
'-main', 'org.antlr.Tool',
'--', '-fo', output_dir
]
for source in self.target.sources:
abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source))
args.append(abs_path)
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(args=args) # TODO: Needs a workunit, when we have a context here.
return True
except (Bootstrapper.Error, Ivy.Error) as e:
print('ANTLR generation failed! %s' % e, file=sys.stderr)
return False
def run_antlrs(self, output_dir):
args = [
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
]
for source in self.target.sources:
abs_path = os.path.abspath(
os.path.join(self.root, self.target.target_base, source))
args.append(abs_path)
try:
ivy = Bootstrapper.default_ivy()
ivy.execute(
args=args
) # TODO: Needs a workunit, when we have a context here.
return True
except (Bootstrapper.Error, Ivy.Error) as e:
print('ANTLR generation failed! %s' % e, file=sys.stderr)
return False
def mapjars(self, genmap, target, executor, workunit_factory=None):
"""
Parameters:
genmap: the jar_dependencies ProductMapping entry for the required products.
target: the target whose jar dependencies are being retrieved.
"""
mapdir = os.path.join(self.mapto_dir(), target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve',
'%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
mapdir,
'-symlink',
]
self.exec_ivy(mapdir, [target],
ivyargs,
confs=target.configurations,
ivy=Bootstrapper.default_ivy(executor),
workunit_factory=workunit_factory,
workunit_name='map-jars')
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
if self.is_mappable_artifact(org, name, f):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf),
confdir).append(f)
genmap.add((org, name, conf),
confdir).append(f)
def mapjars(self, genmap, target, executor, workunit_factory=None):
"""
Parameters:
genmap: the jar_dependencies ProductMapping entry for the required products.
target: the target whose jar dependencies are being retrieved.
"""
mapdir = os.path.join(self.mapto_dir(), target.id)
safe_mkdir(mapdir, clean=True)
ivyargs = [
'-retrieve', '%s/[organisation]/[artifact]/[conf]/'
'[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
'-symlink',
]
self.exec_ivy(mapdir,
[target],
ivyargs,
confs=target.configurations,
ivy=Bootstrapper.default_ivy(executor),
workunit_factory=workunit_factory,
workunit_name='map-jars')
for org in os.listdir(mapdir):
orgdir = os.path.join(mapdir, org)
if os.path.isdir(orgdir):
for name in os.listdir(orgdir):
artifactdir = os.path.join(orgdir, name)
if os.path.isdir(artifactdir):
for conf in os.listdir(artifactdir):
confdir = os.path.join(artifactdir, conf)
for f in os.listdir(confdir):
if self.is_mappable_artifact(org, name, f):
# TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
# conf whitelist
genmap.add(org, confdir).append(f)
genmap.add((org, name), confdir).append(f)
genmap.add(target, confdir).append(f)
genmap.add((target, conf), confdir).append(f)
genmap.add((org, name, conf), confdir).append(f)
def __init__(self, context, confs=None):
super(IvyResolve, self).__init__(context)
work_dir = context.config.get('ivy-resolve', 'workdir')
self._ivy_bootstrapper = Bootstrapper.instance()
self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
self._confs = confs or context.config.getlist('ivy-resolve', 'confs', default=['default'])
self._classpath_dir = os.path.join(work_dir, 'mapped')
self._outdir = context.options.ivy_resolve_outdir or os.path.join(work_dir, 'reports')
self._open = context.options.ivy_resolve_open
self._report = self._open or context.options.ivy_resolve_report
self._ivy_bootstrap_key = 'ivy'
ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan')
self._jvm_tool_bootstrapper.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools)
self._ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
context.products.require_data('exclusives_groups')
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def ivy_resolve(self, targets, executor=None, symlink_ivyxml=False, silent=False,
workunit_name=None, workunit_labels=None):
if executor and not isinstance(executor, Executor):
raise ValueError('The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
targets = set(targets)
if not targets:
return []
work_dir = self.context.config.get('ivy-resolve', 'workdir')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
with self.invalidated(targets,
only_buildfiles=True,
invalidate_dependents=True,
silent=silent) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(work_dir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError('Ivy failed to create classpath file at %s'
% raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home, raw_target_classpath_file,
symlink_dir, target_classpath_file)
with Task.symlink_map_lock:
all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map', lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [path for path in stripped_classpath if ivy_utils.is_classpath_artifact(path)]
def xml_report_path(self, targets, conf):
"""The path to the xml report ivy creates after a retrieve."""
org, name = self.identify(targets)
cachedir = Bootstrapper.instance().ivy_cache_dir
return os.path.join(cachedir, '%s-%s-%s.xml' % (org, name, conf))
def xml_report_path(self, targets, conf): """The path to the xml report ivy creates after a retrieve.""" org, name = self.identify(targets) cachedir = Bootstrapper.instance().ivy_cache_dir return os.path.join(cachedir, '%s-%s-%s.xml' % (org, name, conf))
def ivy_resolve(self,
targets,
executor=None,
symlink_ivyxml=False,
silent=False,
workunit_name=None,
workunit_labels=None):
if executor and not isinstance(executor, Executor):
raise ValueError(
'The executor must be an Executor instance, given %s of type %s'
% (executor, type(executor)))
ivy = Bootstrapper.default_ivy(
java_executor=executor,
bootstrap_workunit_factory=self.context.new_workunit)
targets = set(targets)
if not targets:
return []
work_dir = self.context.config.get('ivy-resolve', 'workdir')
ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
with self.invalidated(targets,
only_buildfiles=True,
invalidate_dependents=True,
silent=silent) as invalidation_check:
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
target_classpath_file = os.path.join(target_workdir, 'classpath')
raw_target_classpath_file = target_classpath_file + '.raw'
raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
# A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
# in artifact-cached analysis files are consistent across systems.
# Note that we have one global, well-known symlink dir, again so that paths are
# consistent across builds.
symlink_dir = os.path.join(work_dir, 'jars')
# Note that it's possible for all targets to be valid but for no classpath file to exist at
# target_classpath_file, e.g., if we previously built a superset of targets.
if invalidation_check.invalid_vts or not os.path.exists(
raw_target_classpath_file):
args = ['-cachepath', raw_target_classpath_file_tmp]
def exec_ivy():
ivy_utils.exec_ivy(
target_workdir=target_workdir,
targets=targets,
args=args,
ivy=ivy,
workunit_name='ivy',
workunit_factory=self.context.new_workunit,
symlink_ivyxml=symlink_ivyxml)
if workunit_name:
with self.context.new_workunit(name=workunit_name,
labels=workunit_labels
or []):
exec_ivy()
else:
exec_ivy()
if not os.path.exists(raw_target_classpath_file_tmp):
raise TaskError(
'Ivy failed to create classpath file at %s' %
raw_target_classpath_file_tmp)
shutil.move(raw_target_classpath_file_tmp,
raw_target_classpath_file)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts,
[raw_target_classpath_file])])
# Make our actual classpath be symlinks, so that the paths are uniform across systems.
# Note that we must do this even if we read the raw_target_classpath_file from the artifact
# cache. If we cache the target_classpath_file we won't know how to create the symlinks.
symlink_map = IvyUtils.symlink_cachepath(self.context.ivy_home,
raw_target_classpath_file,
symlink_dir,
target_classpath_file)
with Task.symlink_map_lock:
all_symlinks_map = self.context.products.get_data(
'symlink_map') or defaultdict(list)
for path, symlink in symlink_map.items():
all_symlinks_map[os.path.realpath(path)].append(symlink)
self.context.products.safe_create_data('symlink_map',
lambda: all_symlinks_map)
with IvyUtils.cachepath(target_classpath_file) as classpath:
stripped_classpath = [path.strip() for path in classpath]
return [
path for path in stripped_classpath
if ivy_utils.is_classpath_artifact(path)
]
