def test_derived_from_chain(self):
context = self.context()
# add concrete target
build_file = self.add_to_build_file(
'y/BUILD',
dedent('''
java_library(
name='concrete',
sources=['SourceA.scala'],
)
'''))
concrete_address = BuildFileAddress(build_file, 'concrete')
context.build_graph.inject_address_closure(concrete_address)
concrete = context.build_graph.get_target(concrete_address)
# add synthetic targets
syn_one = context.add_new_target(SyntheticAddress('y', 'syn_one'),
JavaLibrary,
derived_from=concrete,
sources=["SourceB.scala"])
syn_two = context.add_new_target(SyntheticAddress('y', 'syn_two'),
JavaLibrary,
derived_from=syn_one,
sources=["SourceC.scala"])
# validate
self.assertEquals(list(syn_two.derived_from_chain),
[syn_one, concrete])
self.assertEquals(list(syn_one.derived_from_chain), [concrete])
self.assertEquals(list(concrete.derived_from_chain), [])
def _create_java_target(self, target, dependees):
genfiles = []
for source in target.sources_relative_to_source_root():
path = os.path.join(target.target_base, source)
genfiles.extend(calculate_genfiles(path, source).get('java', []))
spec_path = os.path.relpath(self.java_out, get_buildroot())
address = SyntheticAddress(spec_path, target.id)
deps = OrderedSet(self.javadeps)
import_jars = target.imports
jars_tgt = self.context.add_new_target(SyntheticAddress(spec_path, target.id+str('-rjars')),
JarLibrary,
jars=import_jars,
derived_from=target)
# Add in the 'spec-rjars' target, which contains all the JarDependency targets passed in via the
# imports parameter. Each of these jars is expected to contain .proto files bundled together
# with their .class files.
deps.add(jars_tgt)
tgt = self.context.add_new_target(address,
JavaLibrary,
derived_from=target,
sources=genfiles,
provides=target.provides,
dependencies=deps,
excludes=target.payload.get_field_value('excludes'))
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
def _synthesize_resources_target(self):
# Create an address for the synthetic target.
spec = self.address.spec + '_synthetic_resources'
synthetic_address = SyntheticAddress(spec=spec)
# For safety, ensure an address that's not used already, even though that's highly unlikely.
while self._build_graph.contains_address(synthetic_address):
spec += '_'
synthetic_address = SyntheticAddress(spec=spec)
self._build_graph.inject_synthetic_target(
synthetic_address,
Resources,
sources=self.payload.resources,
derived_from=self)
return self._build_graph.get_target(synthetic_address)
def create_classes_jar_target(self, target, archive, jar_file):
"""Create a JarLibrary target containing the jar_file as a JarDependency.
:param AndroidLibrary target: The new JarLibrary will be derived from this AndroidLibrary .
:param string archive: Archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'.
:param string jar_file: Full path of the classes.jar contained within unpacked aar files.
:return: new_target.
:rtype::class:`pants.backend.jvm.targets.java_library.JarLibrary`
"""
# TODO(mateor) add another JarDependency for every jar under 'libs'.
# Try to parse revision number. This is just to satisfy the spec, the rev is part of 'archive'.
archive_version = os.path.splitext(archive)[0].rpartition('-')[-1]
jar_url = 'file://{0}'.format(jar_file)
jar_dep = JarDependency(org=target.id,
name=archive,
rev=archive_version,
url=jar_url)
address = SyntheticAddress(self.workdir,
'{}-classes.jar'.format(archive))
new_target = self.context.add_new_target(address,
JarLibrary,
jars=[jar_dep],
derived_from=target)
return new_target
def create_python_library(self,
relpath,
name,
source_contents_map=None,
dependencies=(),
provides=None):
sources = ['__init__.py'] + source_contents_map.keys(
) if source_contents_map else None
sources_strs = ["'{0}'".format(s)
for s in sources] if sources else None
self.create_file(relpath=self.build_path(relpath),
contents=dedent("""
python_library(
name='{name}',
{sources_clause}
dependencies=[
{dependencies}
],
{provides_clause}
)
""").format(name=name,
sources_clause='sources=[{0}],'.format(','.join(sources_strs))
if sources_strs else '',
dependencies=','.join(map(repr, dependencies)),
provides_clause='provides={0},'.format(provides)
if provides else ''))
if source_contents_map:
self.create_file(relpath=os.path.join(relpath, '__init__.py'))
for source, contents in source_contents_map.items():
self.create_file(relpath=os.path.join(relpath, source),
contents=contents)
return self.target(SyntheticAddress(relpath, name).spec)
def createtarget(self, lang, gentarget, dependees):
predicates = self.genlangs()
languages = predicates.keys()
if not (lang in languages) or not (predicates[lang](gentarget)):
raise TaskError('Invalid language "{lang}" for task {task}'.format(
lang=lang, task=type(self).__name__))
to_generate = []
for source in gentarget.sources_relative_to_buildroot():
to_generate.extend(
self._sources_to_be_generated(gentarget.package, source))
spec_path = os.path.join(
os.path.relpath(self.workdir, get_buildroot()), 'gen-java')
address = SyntheticAddress(spec_path=spec_path,
target_name=gentarget.id)
target = self.context.add_new_target(
address,
JavaLibrary,
derived_from=gentarget,
sources=to_generate,
provides=gentarget.provides,
dependencies=[],
excludes=gentarget.payload.excludes)
for dependee in dependees:
dependee.inject_dependency(target.address)
return target
def _create_java_target(self, target, dependees):
antlr_files_suffix = ["Lexer.java", "Parser.java"]
if target.compiler == 'antlr4':
antlr_files_suffix = ["BaseListener.java", "BaseVisitor.java",
"Listener.java", "Visitor.java"] + antlr_files_suffix
generated_sources = []
for source in target.sources_relative_to_source_root():
# Antlr enforces that generated sources are relative to the base filename, and that
# each grammar filename must match the resulting grammar Lexer and Parser classes.
source_base, source_ext = os.path.splitext(source)
for suffix in antlr_files_suffix:
generated_sources.append(source_base + suffix)
deps = self._resolve_java_deps(target)
syn_target_sourceroot = os.path.join(self._java_out(target), target.target_base)
spec_path = os.path.relpath(syn_target_sourceroot, get_buildroot())
address = SyntheticAddress(spec_path=spec_path, target_name=target.id)
tgt = self.context.add_new_target(address,
JavaLibrary,
dependencies=deps,
derived_from=target,
sources=generated_sources,
provides=target.provides,
excludes=target.excludes)
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
def create_non_python_target(self, relpath, name):
self.create_file(relpath=self.build_path(relpath),
contents=dedent("""
jvm_target(
name='{name}',
)
""").format(name=name))
return self.target(SyntheticAddress(relpath, name).spec)
def _get_target(spec, build_graph):
try:
address = SyntheticAddress(spec)
except IOError as e:
raise TaskError('Failed to parse address: %s: %s' % (address, e))
match = build_graph.get_target(address)
if not match:
raise TaskError('Invalid target address: %s' % address)
return match
def create_target(files, deps):
spec_path = os.path.join(self.combined_relpath, 'gen-py')
name = '{target_name}'.format(target_name=target.name)
spec = '{spec_path}:{name}'.format(spec_path=spec_path, name=name)
address = SyntheticAddress(spec=spec)
return self.context.add_new_target(address,
PythonLibrary,
derived_from=target,
sources=files,
dependencies=deps)
def prepare(self, round_manager):
super(AaptGen, self).prepare(round_manager)
# prepare exactly N android jar targets where N is the number of SDKs in-play
sdks = set(ar.target_sdk for ar in self.context.targets(predicate=self.is_gentarget))
for sdk in sdks:
jar_url = 'file://{0}'.format(self.android_jar_tool(sdk))
jar = JarDependency(org='com.google', name='android', rev=sdk, url=jar_url)
address = SyntheticAddress(self.workdir, '{0}-jars'.format(sdk))
self._jar_library_by_sdk[sdk] = self.context.add_new_target(address, JarLibrary, jars=[jar])
def provides(self):
if not self.payload.provides:
return None
# TODO(pl): This is an awful hack
if isinstance(self.payload.provides.repo, Compatibility.string):
address = SyntheticAddress(self.payload.provides.repo,
relative_to=self.address.spec_path)
repo_target = self._build_graph.get_target(address)
self.payload.provides.repo = repo_target
return self.payload.provides
def prepare_gen(self, targets):
# prepare exactly N android jar targets where N is the number of SDKs in-play
sdks = set(ar.manifest.target_sdk for ar in targets)
for sdk in sdks:
jar_url = 'file://{0}'.format(self.android_jar_tool(sdk))
jar = JarDependency(org='com.google',
name='android',
rev=sdk,
url=jar_url)
address = SyntheticAddress(self.workdir, '{0}-jars'.format(sdk))
self._jar_library_by_sdk[sdk] = self.context.add_new_target(
address, JarLibrary, jars=[jar])
def provides(self):
if not self._provides:
return None
# TODO(pl): This is an awful hack
for key, binary in self._provides._binaries.iteritems():
if isinstance(binary, Compatibility.string):
address = SyntheticAddress(binary,
relative_to=self.address.spec_path)
self._provides._binaries[key] = self._build_graph.get_target(
address)
return self._provides
def synthetic_target_extra_dependencies(self, target):
# We need to add in the proto imports jars.
jars_address = SyntheticAddress(
os.path.relpath(self.codegen_workdir(target), get_buildroot()),
target.id + '-rjars')
jars_target = self.context.add_new_target(jars_address,
JarLibrary,
jars=target.imported_jars,
derived_from=target)
deps = OrderedSet([jars_target])
deps.update(self.javadeps)
return deps
def create_python_binary(self, relpath, name, entry_point, dependencies=(), provides=None):
self.create_file(relpath=self.build_path(relpath), contents=dedent("""
python_binary(
name='{name}',
entry_point='{entry_point}',
dependencies=[
{dependencies}
],
{provides_clause}
)
""").format(name=name, entry_point=entry_point, dependencies=','.join(map(repr, dependencies)),
provides_clause='provides={0},'.format(provides) if provides else ''))
return self.target(SyntheticAddress(relpath, name).spec)
def createtarget(self, lang, gentarget, dependees):
spec_path = os.path.join(os.path.relpath(self.workdir, get_buildroot()))
address = SyntheticAddress(spec_path=spec_path, target_name=gentarget.id)
aapt_gen_file = self._calculate_genfile(gentarget.package)
deps = OrderedSet([self._jar_library_by_sdk[gentarget.target_sdk]])
tgt = self.context.add_new_target(address,
JavaLibrary,
derived_from=gentarget,
sources=[aapt_gen_file],
dependencies=deps)
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
def create_python_requirement_library(self, relpath, name, requirements):
def make_requirement(req):
return 'python_requirement("{}")'.format(req)
self.create_file(relpath=self.build_path(relpath), contents=dedent("""
python_requirement_library(
name='{name}',
requirements=[
{requirements}
]
)
""").format(name=name, requirements=','.join(map(make_requirement, requirements))))
return self.target(SyntheticAddress(relpath, name).spec)
def _create_python_target(self, target, dependees):
genfiles = []
for source in target.sources_relative_to_source_root():
path = os.path.join(target.target_base, source)
genfiles.extend(calculate_genfiles(path, source).get('py', []))
spec_path = os.path.relpath(self.py_out, get_buildroot())
address = SyntheticAddress(spec_path, target.id)
tgt = self.context.add_new_target(address,
PythonLibrary,
derived_from=target,
sources=genfiles,
dependencies=self.pythondeps)
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
def create_resource_target(self, target, archive, manifest, resource_dir):
"""Create an AndroidResources target.
:param AndroidLibrary target: AndroidLibrary that the new AndroidResources target derives from.
:param string archive: Archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'.
:param string resource_dir: Full path of the res directory contained within aar files.
:return: A new Target.
:rtype: AndroidResources
"""
address = SyntheticAddress(self.workdir, '{}-resources'.format(archive))
new_target = self.context.add_new_target(address, AndroidResources,
manifest=manifest, resource_dir=resource_dir,
derived_from=target)
return new_target
def create_python_binary(self, relpath, name, entry_point,
dependencies=()):
self.create_file(relpath=self.build_path(relpath),
contents=dedent("""
python_binary(
name='{name}',
entry_point='{entry_point}',
dependencies=[
{dependencies}
]
)
""").format(name=name,
entry_point=entry_point,
dependencies=','.join(map(repr, dependencies))))
return self.target(SyntheticAddress(relpath, name).spec)
def create_android_library_target(self, target, archive,
unpacked_aar_location):
"""Create an AndroidLibrary target.
The aar files are unpacked and the contents used to create a new AndroidLibrary target.
:param AndroidLibrary target: AndroidLibrary that the new AndroidLibrary target derives from.
:param string archive: An archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'.
:param string unpacked_aar_location: Full path of dir holding contents of an unpacked aar file.
:return: new_target
:rtype::class:`pants.backend.android.targets.AndroidLibrary`
"""
# The following three elements of an aar file have names mandated by the aar spec:
# http://tools.android.com/tech-docs/new-build-system/aar-format
# They are said to be mandatory although in practice that assumption only holds for manifest.
manifest = os.path.join(unpacked_aar_location, 'AndroidManifest.xml')
jar_file = os.path.join(unpacked_aar_location, 'classes.jar')
resource_dir = os.path.join(unpacked_aar_location, 'res')
# Sanity check to make sure all .aar files we expect to be unpacked are actually unpacked.
if not os.path.isfile(manifest):
raise self.MissingElementException(
"An AndroidManifest.xml is expected in every unpacked "
".aar file but none was found in the {} archive "
"for the {} target".format(archive, target))
# Depending on the contents of the unpacked aar file, create the dependencies.
deps = []
if os.path.isdir(resource_dir):
deps.append(
self.create_resource_target(target, archive, manifest,
resource_dir))
if os.path.isfile(jar_file):
deps.append(
self.create_classes_jar_target(target, archive, jar_file))
address = SyntheticAddress(self.workdir,
'{}-android_library'.format(archive))
new_target = self.context.add_new_target(
address,
AndroidLibrary,
manifest=manifest,
include_patterns=target.include_patterns,
exclude_patterns=target.exclude_patterns,
dependencies=deps,
derived_from=target)
return new_target
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
address = SyntheticAddress(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
def _addargs(self, lines, target, resources_by_target, transitive):
def is_configuration_info(line):
line = line.strip()
return line and not line.startswith('#')
if any(filter(is_configuration_info, lines)):
resource_root = os.path.join(self.workdir, target.id)
safe_mkdir(resource_root)
resource_index = '0' if transitive else '1'
resource_rel_path = os.path.join(
self.RESOURCE_RELDIR, '{0}.{1}'.format(self.RESOURCE_BASENAME,
resource_index))
content = '# Created by pants goal args-apt\n'
content += ''.join(sorted(lines))
resource_path = os.path.join(resource_root, resource_rel_path)
with safe_open(resource_path, 'w') as resource_fp:
resource_fp.write(content)
self.context.log.debug(
'Added args-apt resource file {rel_path} for {target}:\n{content}'
.format(rel_path=os.path.relpath(resource_path,
get_buildroot()),
target=target,
content=content))
# We'll get 2 cmdline resources for binary targets - a 0 transitive and 1 immediate.
# The 0 will always trump at java runtime, so its harmless to attach both resources in
# separate dependencies to keep the implementation here simple; we just need distinct target
# names for each as a result.
target_name = '{0}.transitive'.format(
target.id) if transitive else target.id
spec_path = os.path.relpath(resource_root, get_buildroot())
address = SyntheticAddress(spec_path=spec_path,
target_name=target_name)
args_apt_resources = self.context.add_new_target(
address,
Resources,
derived_from=target,
sources=[resource_rel_path])
target.inject_dependency(address)
resources_by_target[args_apt_resources].add_rel_paths(
resource_root, [resource_rel_path])
def create_sdk_jar_deps(self, binaries):
"""Create a JarLibrary target for every sdk in play.
:param list binaries: A list of AndroidBinary targets.
"""
# Prepare exactly N android jar targets where N is the number of SDKs in-play.
for binary in binaries:
sdk = binary.target_sdk
if sdk not in self._jar_library_by_sdk:
jar_url = 'file://{0}'.format(self.android_jar(binary))
jar = JarDependency(org='com.google',
name='android',
rev=sdk,
url=jar_url)
address = SyntheticAddress(self.workdir,
'android-{0}.jar'.format(sdk))
self._jar_library_by_sdk[sdk] = self.context.add_new_target(
address, JarLibrary, jars=[jar])
def execute_antlr4_test(self, expected_package):
context = self.create_context()
task = self.execute(context)
# get the synthetic target from the private graph
task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java')
syn_sourceroot = os.path.join(task_outdir, self.PARTS['srcroot'])
syn_target_name = ('{srcroot}/{dir}.{name}'.format(
**self.PARTS)).replace('/', '.')
syn_address = SyntheticAddress(spec_path=os.path.relpath(
syn_sourceroot, self.build_root),
target_name=syn_target_name)
syn_target = context.build_graph.get_target(syn_address)
# verify that the synthetic target's list of sources match what are actually created
def re_relativize(p):
"""Take a path relative to task_outdir, and make it relative to the build_root"""
return os.path.relpath(os.path.join(task_outdir, p),
self.build_root)
actual_sources = [
re_relativize(s)
for s in Fileset.rglobs('*.java', root=task_outdir)
]
self.assertEquals(set(syn_target.sources_relative_to_buildroot()),
set(actual_sources))
# and that the synthetic target has a valid source root and the generated sources have the
# expected java package
def get_package(path):
with open(path) as fp:
for line in fp:
match = self.PACKAGE_RE.match(line)
if match:
return match.group('package_name')
return None
for source in syn_target.sources_relative_to_source_root():
source_path = os.path.join(syn_sourceroot, source)
self.assertTrue(
os.path.isfile(source_path),
"{0} is not the source root for {1}".format(
syn_sourceroot, source))
self.assertEqual(expected_package, get_package(source_path))
def _create_java_target(self, target, dependees):
genfiles = []
for source in target.sources_relative_to_source_root():
path = os.path.join(target.target_base, source)
genfiles.extend(calculate_genfiles(path, source).get('java', []))
name = '{target_name}'.format(target_name=target.name)
spec_path = os.path.relpath(self.java_out, get_buildroot())
spec = '{spec_path}:{name}'.format(spec_path=spec_path, name=name)
address = SyntheticAddress(spec=spec)
tgt = self.context.add_new_target(address,
JavaLibrary,
derived_from=target,
sources=genfiles,
provides=target.provides,
dependencies=self.javadeps,
excludes=target.payload.excludes)
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
def inject_address_closure_into_build_graph(self,
address,
build_graph,
addresses_already_closed=None):
addresses_already_closed = addresses_already_closed or set()
if address in addresses_already_closed:
return
self._populate_target_proxy_transitive_closure_for_address(address)
target_proxy = self._target_proxy_by_address[address]
if not build_graph.contains_address(address):
addresses_already_closed.add(address)
for dep_address in target_proxy.dependency_addresses:
self.inject_address_closure_into_build_graph(
dep_address, build_graph, addresses_already_closed)
target = target_proxy.to_target(build_graph)
build_graph.inject_target(
target, dependencies=target_proxy.dependency_addresses)
for traversable_spec in target.traversable_dependency_specs:
spec_path, target_name = parse_spec(
traversable_spec, relative_to=address.spec_path)
self._inject_spec_closure_into_build_graph(
spec_path, target_name, build_graph,
addresses_already_closed)
traversable_spec_target = build_graph.get_target(
SyntheticAddress(spec_path, target_name))
if traversable_spec_target not in target.dependencies:
build_graph.inject_dependency(
dependent=target.address,
dependency=traversable_spec_target.address)
target.mark_transitive_invalidation_hash_dirty()
for traversable_spec in target.traversable_specs:
spec_path, target_name = parse_spec(
traversable_spec, relative_to=address.spec_path)
self._inject_spec_closure_into_build_graph(
spec_path, target_name, build_graph,
addresses_already_closed)
target.mark_transitive_invalidation_hash_dirty()
def create_target(self, binary, gentarget):
"""Create a JavaLibrary target for the R.java files created by the aapt tool.
:param AndroidBinary binary: AndroidBinary target whose target_sdk is used.
:param AndroidTarget gentarget: AndroidBinary or Library that owns the processed resources.
:returns new_target: Synthetic target for the R.java output of the aapt tool.
:rtype::class:`pants.backend.jvm.targets.java_library.JavaLibrary`
"""
spec_path = os.path.join(
os.path.relpath(self.aapt_out(binary), get_buildroot()))
address = SyntheticAddress(spec_path=spec_path,
target_name=gentarget.id)
deps = [self._jar_library_by_sdk[binary.target_sdk]]
new_target = self.context.add_new_target(
address,
JavaLibrary,
derived_from=gentarget,
sources=[self._relative_genfile(gentarget)],
dependencies=deps)
return new_target
def _create_java_target(self, target, dependees):
genfiles = []
for source in target.sources_relative_to_source_root():
path = os.path.join(target.target_base, source)
genfiles.extend(calculate_genfiles(
path,
source,
target.payload.service_writer).get('java', []))
spec_path = os.path.relpath(self.java_out, get_buildroot())
address = SyntheticAddress(spec_path, target.id)
deps = OrderedSet(self.javadeps)
tgt = self.context.add_new_target(address,
JavaLibrary,
derived_from=target,
sources=genfiles,
provides=target.provides,
dependencies=deps,
excludes=target.payload.excludes)
for dependee in dependees:
dependee.inject_dependency(tgt.address)
return tgt
