def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
if JvmResolveSubsystem.global_instance().get_options().resolver != 'ivy':
return
compile_classpath = self.context.products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
targets = self.context.targets()
if all(not isinstance(target, JarLibrary) for target in targets):
if self._report:
self.context.log.info("Not generating a report. No resolution performed.")
return
executor = self.create_java_executor()
results = self.resolve(executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=self.get_options().confs,
extra_args=self._args)
if self._report:
results_with_resolved_artifacts = filter(lambda r: r.has_resolved_artifacts, results)
if not results_with_resolved_artifacts:
self.context.log.info("Not generating a report. No resolution performed.")
else:
for result in results_with_resolved_artifacts:
self._generate_ivy_report(result)
def execute(self):
targets = self.context.targets(self._is_remote)
runtime_classpath_product = self.context.products.get_data(
'runtime_classpath', init_func=ClasspathProducts.init_func(self.get_options().pants_workdir)
)
with self.invalidated(targets, invalidate_dependents=True, topological_order=True) as invalidation_check:
# The fetches are idempotent operations from the subsystem, invalidation only controls recreating the symlinks.
for vt in invalidation_check.all_vts:
remote_source = RemoteSourceFetcher(vt.target)
fetched = remote_source.path
safe_mkdir(fetched)
# Some unfortunate rigamorole to cover for the case where different targets want the same fetched file
# but extracted/not. Both cases use the same base namespacing so we rely on the target to tell us.
fetch_dir = fetched if remote_source.extracted else os.path.dirname(fetched)
filenames = os.listdir(fetched) if remote_source.extracted else [os.path.basename(fetched)]
stable_outpath = vt.target.namespace + '-{}'.format('extracted') if vt.target.extract else ''
for filename in filenames:
symlink_file = os.path.join(vt.results_dir, filename)
if not vt.valid or not os.path.isfile(symlink_file):
safe_rmtree(symlink_file)
relative_symlink(os.path.join(fetch_dir, filename), symlink_file)
self.context.products.get('remote_files').add(vt.target, vt.results_dir).append(filename)
# The runtime_classpath product is a constructed object that is rooted in the results_dir.
runtime_classpath_product.add_for_target(vt.target, [('default', vt.results_dir)])
def _populate_compile_classpath(self):
products = self.context.products
compile_classpath = products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
sorted_targets = sorted(
self.context.targets(predicate=lambda t: t in self.all_jar_libs),
key=lambda t: t.address.spec,
)
for target in sorted_targets:
resolved_jars = []
for coord in sorted(self.target_to_maven_coordinate_closure[target.id]):
m2_coord = M2Coordinate(
org=coord.groupId,
name=coord.artifactId,
rev=coord.version,
classifier=coord.classifier,
ext=coord.packaging,
)
sorted_artifact_paths = sorted(
artifact.artifact_path
for artifact in self.maven_coordinate_to_provided_artifacts[coord]
)
for artifact_path in sorted_artifact_paths:
resolved_jar = ResolvedJar(
coordinate=m2_coord,
pants_path=os.path.join(self.artifact_symlink_dir, artifact_path),
cache_path=os.path.join(self.pom_cache_dir, artifact_path),
)
resolved_jars.append(resolved_jar)
compile_classpath.add_jars_for_targets([target], 'default', resolved_jars)
def execute(self):
node_paths = self.context.products.get_data(NodePaths)
runtime_classpath_product = self.context.products.get_data(
'runtime_classpath', init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
bundleable_js_product = self.context.products.get_data(
'bundleable_js', init_func=lambda: defaultdict(MultipleRootedProducts))
targets = self.context.targets(predicate=self.is_node_module)
with self.invalidated(targets, invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.all_vts:
target = vt.target
node_installed_path = node_paths.node_path(target)
with pushd(node_installed_path):
if not vt.valid:
self._run_build_script(target, vt.results_dir, node_installed_path)
if not target.payload.dev_dependency:
output_dir = self._get_output_dir(target, node_installed_path)
# Make sure that there is output generated.
if not os.path.exists(output_dir):
raise TaskError(
'Target {} has build script {} specified, but did not generate any output '
'at {}.\n'.format(
target.address.reference(), target.payload.build_script, output_dir))
absolute_symlink(output_dir, os.path.join(vt.results_dir, target.address.target_name))
bundleable_js_product[target].add_abs_paths(output_dir, [output_dir])
runtime_classpath_product.add_for_target(target, [('default', vt.results_dir)])
def execute_export(self, *specs):
context = self.context(target_roots=[self.target(spec) for spec in specs])
context.products.safe_create_data(
"compile_classpath", init_func=ClasspathProducts.init_func(self.pants_workdir)
)
task = self.create_task(context)
return list(task.console_output(list(task.context.targets()), context.products.get_data("compile_classpath")))
def test_incremental_caching(self):
with temporary_dir(root_dir=self.pants_workdir) as jar_dir, \
temporary_dir(root_dir=self.pants_workdir) as dist_dir:
self.set_options(pants_distdir=dist_dir)
target = self.make_target(
'java/classpath:java_lib',
target_type=JavaLibrary,
sources=['com/foo/Bar.java'],
)
context = self.context(target_roots=[target])
runtime_classpath = context.products.get_data('runtime_classpath',
init_func=ClasspathProducts.init_func(self.pants_workdir))
task = self.create_task(context)
target_classpath_output = os.path.join(dist_dir, self.options_scope)
# Create a classpath entry.
touch(os.path.join(jar_dir, 'z1.jar'))
runtime_classpath.add_for_target(target, [(self.DEFAULT_CONF, os.path.join(jar_dir, 'z1.jar'))])
task.execute()
# Check only one symlink and classpath.txt were created.
self.assertEqual(len(os.listdir(target_classpath_output)), 2)
self.assertEqual(
os.path.realpath(os.path.join(target_classpath_output,
sorted(os.listdir(target_classpath_output))[0])),
os.path.join(jar_dir, 'z1.jar')
)
# Remove the classpath entry.
runtime_classpath.remove_for_target(target, [(self.DEFAULT_CONF, os.path.join(jar_dir, 'z1.jar'))])
# Add a different classpath entry
touch(os.path.join(jar_dir, 'z2.jar'))
runtime_classpath.add_for_target(target, [(self.DEFAULT_CONF, os.path.join(jar_dir, 'z2.jar'))])
task.execute()
# Check the symlink was updated.
self.assertEqual(len(os.listdir(target_classpath_output)), 2)
self.assertEqual(
os.path.realpath(os.path.join(target_classpath_output,
sorted(os.listdir(target_classpath_output))[0])),
os.path.join(jar_dir, 'z2.jar')
)
# Add a different classpath entry
touch(os.path.join(jar_dir, 'z3.jar'))
runtime_classpath.add_for_target(target, [(self.DEFAULT_CONF, os.path.join(jar_dir, 'z3.jar'))])
task.execute()
self.assertEqual(len(os.listdir(target_classpath_output)), 3)
classpath = sorted(os.listdir(target_classpath_output))[2]
with safe_open(os.path.join(target_classpath_output, classpath)) as classpath_file:
# Assert there is only one line ending with a newline
self.assertListEqual(
classpath_file.readlines(),
[
os.pathsep.join([os.path.join(jar_dir, 'z2.jar'), os.path.join(jar_dir, 'z3.jar')]) + '\n'
]
)
def ensure_classpath_products(self, context):
"""Gets or creates the classpath products expected by `JvmBinaryTask`.
:param context: The pants run context to get/create/associate classpath products with.
:type context: :class:`pants.goal.context.Context`
:returns: The classpath products associated with the given `context`
:rtype: :class:`pants.backend.jvm.tasks.classpath_products.ClasspathProducts`
"""
return context.products.get_data('runtime_classpath', init_func=ClasspathProducts.init_func(self.pants_workdir))
def execute(self):
cp_init_func = ClasspathProducts.init_func(self.get_options().pants_workdir)
compile_classpath = self.context.products.get_data('compile_classpath', init_func=cp_init_func)
with self.invalidated(self.context.targets(is_tools_jar)) as invalidation_check:
for vt in invalidation_check.all_vts:
jar_path = self._jar_path(vt.results_dir)
if not vt.valid:
self._symlink_tools_jar(jar_path)
compile_classpath.add_for_target(vt.target, [('default', jar_path)])
def test_if_runtime_classpath_exists(self):
target = self.make_target(
'java/classpath:java_lib',
target_type=JavaLibrary,
sources=['com/foo/Bar.java'],
)
context = self.context(target_roots=[target])
compile_classpath = context.products.get_data('compile_classpath', ClasspathProducts.init_func(self.pants_workdir))
compile_entry = os.path.join(self.pants_workdir, 'compile-entry')
pre_init_runtime_entry = os.path.join(self.pants_workdir, 'pre-inited-runtime-entry')
compile_classpath.add_for_targets([target], [('default', compile_entry)])
runtime_classpath = context.products.get_data('runtime_classpath', ClasspathProducts.init_func(self.pants_workdir))
runtime_classpath.add_for_targets([target], [('default', pre_init_runtime_entry)])
task = self.create_task(context)
resulting_classpath = task.create_runtime_classpath()
self.assertEqual([('default', pre_init_runtime_entry), ('default', compile_entry)],
resulting_classpath.get_for_target(target))
def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
executor = self.create_java_executor()
targets = self.context.targets()
compile_classpath = self.context.products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
resolve_hash_name = self.resolve(executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=self.confs,
extra_args=self._args)
if self._report:
self._generate_ivy_report(resolve_hash_name)
def resolve_jars(self, targets):
"""Resolve
:param targets: targets that have dependencies to resolve
:return: structure containing the path to resolved jars
:rtype: ClasspathProducts
"""
executor = SubprocessExecutor(DistributionLocator.cached())
classpath_products = self.context.products.get_data('classpath_products',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir))
self.resolve(executor=executor,
targets=targets,
classpath_products=classpath_products,
confs=['default'],
extra_args=())
return classpath_products
def _setup(self, target_classfiles):
"""Takes a dict mapping targets to lists of classfiles."""
context = self.context(target_roots=target_classfiles.keys())
# Create classfiles in a target-specific directory, and add it to the classpath for the target.
classpath_products = context.products.get_data('runtime_classpath', ClasspathProducts.init_func(self.pants_workdir))
for target, classfiles in target_classfiles.items():
target_dir = os.path.join(self.test_workdir, target.id)
safe_mkdir(target_dir)
for classfile in classfiles:
touch(os.path.join(target_dir, classfile))
classpath_products.add_for_target(target, [('default', target_dir)])
product_deps_by_src = context.products.get_data('product_deps_by_src', dict)
return self.create_task(context), product_deps_by_src
def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
jvm_resolve_subsystem = JvmResolveSubsystem.global_instance()
if jvm_resolve_subsystem.get_options().resolver != 'coursier':
return
# executor = self.create_java_executor()
classpath_products = self.context.products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(
self.get_options().pants_workdir))
executor = self.create_java_executor()
self.resolve(self.context.targets(), classpath_products, sources=False, javadoc=False, executor=executor)
def execute_export(self, *specs, **options_overrides):
options = {
JvmPlatform.options_scope: {
"default_platform": "java6",
"platforms": {"java6": {"source": "1.6", "target": "1.6"}},
}
}
options.update(options_overrides)
context = self.context(
options=options, target_roots=[self.target(spec) for spec in specs], for_subsystems=[JvmPlatform]
)
context.products.safe_create_data(
"compile_classpath", init_func=ClasspathProducts.init_func(self.pants_workdir)
)
task = self.create_task(context)
return list(task.console_output(list(task.context.targets()), context.products.get_data("compile_classpath")))
def execute_export(self, *specs, **options_overrides):
options = {
JvmPlatform.options_scope: {
'default_platform': 'java6',
'platforms': {
'java6': {'source': '1.6', 'target': '1.6'}
}
},
}
options.update(options_overrides)
context = self.context(options=options, target_roots=[self.target(spec) for spec in specs],
for_subsystems=[JvmPlatform])
context.products.safe_create_data('compile_classpath',
init_func=ClasspathProducts.init_func(self.pants_workdir))
task = self.create_task(context)
return list(task.console_output(list(task.context.targets()),
context.products.get_data('compile_classpath')))
def test_resolve_specific_with_sources_javadocs(self):
# Create a jar_library with a single dep, and another library with no deps.
dep = JarDependency('commons-lang', 'commons-lang', '2.5')
jar_lib = self.make_target('//:a', JarLibrary, jars=[dep])
scala_lib = self.make_target('//:b', JavaLibrary, sources=[])
with self._temp_workdir() as workdir:
# Confirm that the deps were added to the appropriate targets.
context = self.context(target_roots=[jar_lib, scala_lib])
task = self.prepare_execute(context)
compile_classpath = context.products.get_data('compile_classpath',
init_func=ClasspathProducts.init_func(workdir)
)
task.resolve([jar_lib, scala_lib], compile_classpath, sources=True, javadoc=True, executor=None)
# Both javadoc and sources jars are added to the classpath product
self.assertEqual(['default', 'src_doc', 'src_doc'],
sorted([c[0] for c in compile_classpath.get_for_target(jar_lib)]))
self.assertEqual(0, len(compile_classpath.get_for_target(scala_lib)))
def add_consolidated_bundle(self, context, tgt, files_dict):
"""Add a bundle to the classpath as if it has been consolidated already.
"""
consolidated_classpath = context.products.get_data(
'consolidated_classpath',
init_func=ClasspathProducts.init_func(self.pants_workdir)
)
# Create a temporary directory under the target id, then dump all files.
target_dir = os.path.join(self.test_workdir, tgt.id)
safe_mkdir(target_dir)
entry_path = safe_mkdtemp(dir=target_dir)
classpath_dir = safe_mkdtemp(dir=target_dir)
for rel_path, content in files_dict.items():
safe_file_dump(os.path.join(entry_path, rel_path), content)
# Create Jar to mimic consolidate classpath behavior.
jarpath = os.path.join(classpath_dir, 'output-0.jar')
with self.task.open_jar(jarpath, overwrite=True, compressed=False) as jar:
jar.write(entry_path)
consolidated_classpath.add_for_target(tgt, [('default', jarpath)])
def execute(self):
executor = self.create_java_executor()
targets = sorted(set(self.context.targets()) | self.bag_target_closure)
compile_classpath = self.context.products.get_data(
'compile_classpath',
init_func=ClasspathProducts.init_func(self.get_options().pants_workdir)
)
results = self.resolve(
executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=self.get_options().confs,
extra_args=self._args,
)
if self._report:
results_with_resolved_artifacts = filter(lambda r: r.has_resolved_artifacts, results)
if not results_with_resolved_artifacts:
self.context.log.info("Not generating a report. No resolution performed.")
else:
for result in results_with_resolved_artifacts:
self._generate_ivy_report(result)
def get_runtime_classpath(self, context):
return context.products.get_data('runtime_classpath', init_func=ClasspathProducts.init_func(self.pants_workdir))
def init_products(self, context):
context.products.get_data('compile_classpath', ClasspathProducts.init_func(self.pants_workdir))
context.products.get_data('runtime_classpath', ClasspathProducts.init_func(self.pants_workdir))
def init_products(self, context):
context.products.get_data('compile_classpath', ClasspathProducts.init_func(self.pants_workdir))
context.products.get_data('runtime_classpath', ClasspathProducts.init_func(self.pants_workdir))
def get_runtime_classpath(self, context):
"""
:API: public
"""
return context.products.get_data('runtime_classpath', init_func=ClasspathProducts.init_func(self.pants_workdir))
def prep_before_export(self, context):
context.products.safe_create_data(
"runtime_classpath",
init_func=ClasspathProducts.init_func(self.pants_workdir))
def create_resources_task(self, target_roots=None, **options):
self.set_options(**options)
context = self.context(target_roots=target_roots)
context.products.safe_create_data('compile_classpath', init_func=ClasspathProducts.init_func(self.pants_workdir))
return self.create_task(context)
def test_metacp_job_scheduled_for_jar_library(self):
# Init dependencies for scala library targets.
init_subsystem(
ScalaPlatform,
{ScalaPlatform.options_scope: {
'version': 'custom',
'suffix_version': '2.12',
}}
)
self.make_target(
'//:scala-library',
target_type=JarLibrary,
jars=[JarDependency(org='com.example', name='scala', rev='0.0.0')]
)
jar_target = self.make_target(
'java/classpath:jar_lib',
target_type=JarLibrary,
jars=[JarDependency(org='com.example', name='example', rev='0.0.0')]
)
java_target = self.make_target(
'java/classpath:java_lib',
target_type=JavaLibrary,
sources=['com/example/Foo.java'],
dependencies=[jar_target]
)
scala_target = self.make_target(
'java/classpath:scala_lib',
target_type=ScalaLibrary,
sources=['com/example/Foo.scala'],
dependencies=[jar_target]
)
context = self.context(target_roots=[jar_target])
context.products.get_data('compile_classpath', ClasspathProducts.init_func(self.pants_workdir))
context.products.get_data('runtime_classpath', ClasspathProducts.init_func(self.pants_workdir))
task = self.create_task(context)
# tried for options, but couldn't get it to reconfig
task._size_estimator = lambda srcs: 0
with temporary_dir() as tmp_dir:
compile_contexts = {target: task.create_compile_context(target, os.path.join(tmp_dir, target.id))
for target in [jar_target, java_target, scala_target]}
invalid_targets = [java_target, scala_target, jar_target]
jobs = task._create_compile_jobs(compile_contexts,
invalid_targets,
invalid_vts=[LightWeightVTS(t) for t in invalid_targets],
classpath_product=None)
exec_graph = ExecutionGraph(jobs, task.get_options().print_exception_stacktrace)
dependee_graph = exec_graph.format_dependee_graph()
self.assertEqual(dedent("""
compile_against_rsc(java/classpath:java_lib) -> {}
rsc(java/classpath:scala_lib) -> {
compile_against_rsc(java/classpath:scala_lib)
}
compile_against_rsc(java/classpath:scala_lib) -> {}
metacp(java/classpath:jar_lib) -> {
rsc(java/classpath:scala_lib)
}""").strip(),
dependee_graph)
