def test_derived_from_chain(self): context = self.context() # add concrete target build_file = self.add_to_build_file( 'y/BUILD', dedent(''' java_library( name='concrete', sources=['SourceA.scala'], ) ''')) concrete_address = BuildFileAddress(build_file, 'concrete') context.build_graph.inject_address_closure(concrete_address) concrete = context.build_graph.get_target(concrete_address) # add synthetic targets syn_one = context.add_new_target(SyntheticAddress('y', 'syn_one'), JavaLibrary, derived_from=concrete, sources=["SourceB.scala"]) syn_two = context.add_new_target(SyntheticAddress('y', 'syn_two'), JavaLibrary, derived_from=syn_one, sources=["SourceC.scala"]) # validate self.assertEquals(list(syn_two.derived_from_chain), [syn_one, concrete]) self.assertEquals(list(syn_one.derived_from_chain), [concrete]) self.assertEquals(list(concrete.derived_from_chain), [])
def _create_java_target(self, target, dependees): genfiles = [] for source in target.sources_relative_to_source_root(): path = os.path.join(target.target_base, source) genfiles.extend(calculate_genfiles(path, source).get('java', [])) spec_path = os.path.relpath(self.java_out, get_buildroot()) address = SyntheticAddress(spec_path, target.id) deps = OrderedSet(self.javadeps) import_jars = target.imports jars_tgt = self.context.add_new_target(SyntheticAddress(spec_path, target.id+str('-rjars')), JarLibrary, jars=import_jars, derived_from=target) # Add in the 'spec-rjars' target, which contains all the JarDependency targets passed in via the # imports parameter. Each of these jars is expected to contain .proto files bundled together # with their .class files. deps.add(jars_tgt) tgt = self.context.add_new_target(address, JavaLibrary, derived_from=target, sources=genfiles, provides=target.provides, dependencies=deps, excludes=target.payload.get_field_value('excludes')) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt
def _synthesize_resources_target(self): # Create an address for the synthetic target. spec = self.address.spec + '_synthetic_resources' synthetic_address = SyntheticAddress(spec=spec) # For safety, ensure an address that's not used already, even though that's highly unlikely. while self._build_graph.contains_address(synthetic_address): spec += '_' synthetic_address = SyntheticAddress(spec=spec) self._build_graph.inject_synthetic_target( synthetic_address, Resources, sources=self.payload.resources, derived_from=self) return self._build_graph.get_target(synthetic_address)
def create_classes_jar_target(self, target, archive, jar_file): """Create a JarLibrary target containing the jar_file as a JarDependency. :param AndroidLibrary target: The new JarLibrary will be derived from this AndroidLibrary . :param string archive: Archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'. :param string jar_file: Full path of the classes.jar contained within unpacked aar files. :return: new_target. :rtype::class:`pants.backend.jvm.targets.java_library.JarLibrary` """ # TODO(mateor) add another JarDependency for every jar under 'libs'. # Try to parse revision number. This is just to satisfy the spec, the rev is part of 'archive'. archive_version = os.path.splitext(archive)[0].rpartition('-')[-1] jar_url = 'file://{0}'.format(jar_file) jar_dep = JarDependency(org=target.id, name=archive, rev=archive_version, url=jar_url) address = SyntheticAddress(self.workdir, '{}-classes.jar'.format(archive)) new_target = self.context.add_new_target(address, JarLibrary, jars=[jar_dep], derived_from=target) return new_target
def create_python_library(self, relpath, name, source_contents_map=None, dependencies=(), provides=None): sources = ['__init__.py'] + source_contents_map.keys( ) if source_contents_map else None sources_strs = ["'{0}'".format(s) for s in sources] if sources else None self.create_file(relpath=self.build_path(relpath), contents=dedent(""" python_library( name='{name}', {sources_clause} dependencies=[ {dependencies} ], {provides_clause} ) """).format(name=name, sources_clause='sources=[{0}],'.format(','.join(sources_strs)) if sources_strs else '', dependencies=','.join(map(repr, dependencies)), provides_clause='provides={0},'.format(provides) if provides else '')) if source_contents_map: self.create_file(relpath=os.path.join(relpath, '__init__.py')) for source, contents in source_contents_map.items(): self.create_file(relpath=os.path.join(relpath, source), contents=contents) return self.target(SyntheticAddress(relpath, name).spec)
def createtarget(self, lang, gentarget, dependees): predicates = self.genlangs() languages = predicates.keys() if not (lang in languages) or not (predicates[lang](gentarget)): raise TaskError('Invalid language "{lang}" for task {task}'.format( lang=lang, task=type(self).__name__)) to_generate = [] for source in gentarget.sources_relative_to_buildroot(): to_generate.extend( self._sources_to_be_generated(gentarget.package, source)) spec_path = os.path.join( os.path.relpath(self.workdir, get_buildroot()), 'gen-java') address = SyntheticAddress(spec_path=spec_path, target_name=gentarget.id) target = self.context.add_new_target( address, JavaLibrary, derived_from=gentarget, sources=to_generate, provides=gentarget.provides, dependencies=[], excludes=gentarget.payload.excludes) for dependee in dependees: dependee.inject_dependency(target.address) return target
def _create_java_target(self, target, dependees): antlr_files_suffix = ["Lexer.java", "Parser.java"] if target.compiler == 'antlr4': antlr_files_suffix = ["BaseListener.java", "BaseVisitor.java", "Listener.java", "Visitor.java"] + antlr_files_suffix generated_sources = [] for source in target.sources_relative_to_source_root(): # Antlr enforces that generated sources are relative to the base filename, and that # each grammar filename must match the resulting grammar Lexer and Parser classes. source_base, source_ext = os.path.splitext(source) for suffix in antlr_files_suffix: generated_sources.append(source_base + suffix) deps = self._resolve_java_deps(target) syn_target_sourceroot = os.path.join(self._java_out(target), target.target_base) spec_path = os.path.relpath(syn_target_sourceroot, get_buildroot()) address = SyntheticAddress(spec_path=spec_path, target_name=target.id) tgt = self.context.add_new_target(address, JavaLibrary, dependencies=deps, derived_from=target, sources=generated_sources, provides=target.provides, excludes=target.excludes) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt
def create_non_python_target(self, relpath, name): self.create_file(relpath=self.build_path(relpath), contents=dedent(""" jvm_target( name='{name}', ) """).format(name=name)) return self.target(SyntheticAddress(relpath, name).spec)
def _get_target(spec, build_graph): try: address = SyntheticAddress(spec) except IOError as e: raise TaskError('Failed to parse address: %s: %s' % (address, e)) match = build_graph.get_target(address) if not match: raise TaskError('Invalid target address: %s' % address) return match
def create_target(files, deps): spec_path = os.path.join(self.combined_relpath, 'gen-py') name = '{target_name}'.format(target_name=target.name) spec = '{spec_path}:{name}'.format(spec_path=spec_path, name=name) address = SyntheticAddress(spec=spec) return self.context.add_new_target(address, PythonLibrary, derived_from=target, sources=files, dependencies=deps)
def prepare(self, round_manager): super(AaptGen, self).prepare(round_manager) # prepare exactly N android jar targets where N is the number of SDKs in-play sdks = set(ar.target_sdk for ar in self.context.targets(predicate=self.is_gentarget)) for sdk in sdks: jar_url = 'file://{0}'.format(self.android_jar_tool(sdk)) jar = JarDependency(org='com.google', name='android', rev=sdk, url=jar_url) address = SyntheticAddress(self.workdir, '{0}-jars'.format(sdk)) self._jar_library_by_sdk[sdk] = self.context.add_new_target(address, JarLibrary, jars=[jar])
def provides(self): if not self.payload.provides: return None # TODO(pl): This is an awful hack if isinstance(self.payload.provides.repo, Compatibility.string): address = SyntheticAddress(self.payload.provides.repo, relative_to=self.address.spec_path) repo_target = self._build_graph.get_target(address) self.payload.provides.repo = repo_target return self.payload.provides
def prepare_gen(self, targets): # prepare exactly N android jar targets where N is the number of SDKs in-play sdks = set(ar.manifest.target_sdk for ar in targets) for sdk in sdks: jar_url = 'file://{0}'.format(self.android_jar_tool(sdk)) jar = JarDependency(org='com.google', name='android', rev=sdk, url=jar_url) address = SyntheticAddress(self.workdir, '{0}-jars'.format(sdk)) self._jar_library_by_sdk[sdk] = self.context.add_new_target( address, JarLibrary, jars=[jar])
def provides(self): if not self._provides: return None # TODO(pl): This is an awful hack for key, binary in self._provides._binaries.iteritems(): if isinstance(binary, Compatibility.string): address = SyntheticAddress(binary, relative_to=self.address.spec_path) self._provides._binaries[key] = self._build_graph.get_target( address) return self._provides
def synthetic_target_extra_dependencies(self, target): # We need to add in the proto imports jars. jars_address = SyntheticAddress( os.path.relpath(self.codegen_workdir(target), get_buildroot()), target.id + '-rjars') jars_target = self.context.add_new_target(jars_address, JarLibrary, jars=target.imported_jars, derived_from=target) deps = OrderedSet([jars_target]) deps.update(self.javadeps) return deps
def create_python_binary(self, relpath, name, entry_point, dependencies=(), provides=None): self.create_file(relpath=self.build_path(relpath), contents=dedent(""" python_binary( name='{name}', entry_point='{entry_point}', dependencies=[ {dependencies} ], {provides_clause} ) """).format(name=name, entry_point=entry_point, dependencies=','.join(map(repr, dependencies)), provides_clause='provides={0},'.format(provides) if provides else '')) return self.target(SyntheticAddress(relpath, name).spec)
def createtarget(self, lang, gentarget, dependees): spec_path = os.path.join(os.path.relpath(self.workdir, get_buildroot())) address = SyntheticAddress(spec_path=spec_path, target_name=gentarget.id) aapt_gen_file = self._calculate_genfile(gentarget.package) deps = OrderedSet([self._jar_library_by_sdk[gentarget.target_sdk]]) tgt = self.context.add_new_target(address, JavaLibrary, derived_from=gentarget, sources=[aapt_gen_file], dependencies=deps) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt
def create_python_requirement_library(self, relpath, name, requirements): def make_requirement(req): return 'python_requirement("{}")'.format(req) self.create_file(relpath=self.build_path(relpath), contents=dedent(""" python_requirement_library( name='{name}', requirements=[ {requirements} ] ) """).format(name=name, requirements=','.join(map(make_requirement, requirements)))) return self.target(SyntheticAddress(relpath, name).spec)
def _create_python_target(self, target, dependees): genfiles = [] for source in target.sources_relative_to_source_root(): path = os.path.join(target.target_base, source) genfiles.extend(calculate_genfiles(path, source).get('py', [])) spec_path = os.path.relpath(self.py_out, get_buildroot()) address = SyntheticAddress(spec_path, target.id) tgt = self.context.add_new_target(address, PythonLibrary, derived_from=target, sources=genfiles, dependencies=self.pythondeps) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt
def create_resource_target(self, target, archive, manifest, resource_dir): """Create an AndroidResources target. :param AndroidLibrary target: AndroidLibrary that the new AndroidResources target derives from. :param string archive: Archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'. :param string resource_dir: Full path of the res directory contained within aar files. :return: A new Target. :rtype: AndroidResources """ address = SyntheticAddress(self.workdir, '{}-resources'.format(archive)) new_target = self.context.add_new_target(address, AndroidResources, manifest=manifest, resource_dir=resource_dir, derived_from=target) return new_target
def create_python_binary(self, relpath, name, entry_point, dependencies=()): self.create_file(relpath=self.build_path(relpath), contents=dedent(""" python_binary( name='{name}', entry_point='{entry_point}', dependencies=[ {dependencies} ] ) """).format(name=name, entry_point=entry_point, dependencies=','.join(map(repr, dependencies)))) return self.target(SyntheticAddress(relpath, name).spec)
def create_android_library_target(self, target, archive, unpacked_aar_location): """Create an AndroidLibrary target. The aar files are unpacked and the contents used to create a new AndroidLibrary target. :param AndroidLibrary target: AndroidLibrary that the new AndroidLibrary target derives from. :param string archive: An archive name as fetched by ivy, e.g. 'org.pantsbuild.example-1.0.aar'. :param string unpacked_aar_location: Full path of dir holding contents of an unpacked aar file. :return: new_target :rtype::class:`pants.backend.android.targets.AndroidLibrary` """ # The following three elements of an aar file have names mandated by the aar spec: # http://tools.android.com/tech-docs/new-build-system/aar-format # They are said to be mandatory although in practice that assumption only holds for manifest. manifest = os.path.join(unpacked_aar_location, 'AndroidManifest.xml') jar_file = os.path.join(unpacked_aar_location, 'classes.jar') resource_dir = os.path.join(unpacked_aar_location, 'res') # Sanity check to make sure all .aar files we expect to be unpacked are actually unpacked. if not os.path.isfile(manifest): raise self.MissingElementException( "An AndroidManifest.xml is expected in every unpacked " ".aar file but none was found in the {} archive " "for the {} target".format(archive, target)) # Depending on the contents of the unpacked aar file, create the dependencies. deps = [] if os.path.isdir(resource_dir): deps.append( self.create_resource_target(target, archive, manifest, resource_dir)) if os.path.isfile(jar_file): deps.append( self.create_classes_jar_target(target, archive, jar_file)) address = SyntheticAddress(self.workdir, '{}-android_library'.format(archive)) new_target = self.context.add_new_target( address, AndroidLibrary, manifest=manifest, include_patterns=target.include_patterns, exclude_patterns=target.exclude_patterns, dependencies=deps, derived_from=target) return new_target
def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, **kwargs): address = SyntheticAddress(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target
def _addargs(self, lines, target, resources_by_target, transitive): def is_configuration_info(line): line = line.strip() return line and not line.startswith('#') if any(filter(is_configuration_info, lines)): resource_root = os.path.join(self.workdir, target.id) safe_mkdir(resource_root) resource_index = '0' if transitive else '1' resource_rel_path = os.path.join( self.RESOURCE_RELDIR, '{0}.{1}'.format(self.RESOURCE_BASENAME, resource_index)) content = '# Created by pants goal args-apt\n' content += ''.join(sorted(lines)) resource_path = os.path.join(resource_root, resource_rel_path) with safe_open(resource_path, 'w') as resource_fp: resource_fp.write(content) self.context.log.debug( 'Added args-apt resource file {rel_path} for {target}:\n{content}' .format(rel_path=os.path.relpath(resource_path, get_buildroot()), target=target, content=content)) # We'll get 2 cmdline resources for binary targets - a 0 transitive and 1 immediate. # The 0 will always trump at java runtime, so its harmless to attach both resources in # separate dependencies to keep the implementation here simple; we just need distinct target # names for each as a result. target_name = '{0}.transitive'.format( target.id) if transitive else target.id spec_path = os.path.relpath(resource_root, get_buildroot()) address = SyntheticAddress(spec_path=spec_path, target_name=target_name) args_apt_resources = self.context.add_new_target( address, Resources, derived_from=target, sources=[resource_rel_path]) target.inject_dependency(address) resources_by_target[args_apt_resources].add_rel_paths( resource_root, [resource_rel_path])
def create_sdk_jar_deps(self, binaries): """Create a JarLibrary target for every sdk in play. :param list binaries: A list of AndroidBinary targets. """ # Prepare exactly N android jar targets where N is the number of SDKs in-play. for binary in binaries: sdk = binary.target_sdk if sdk not in self._jar_library_by_sdk: jar_url = 'file://{0}'.format(self.android_jar(binary)) jar = JarDependency(org='com.google', name='android', rev=sdk, url=jar_url) address = SyntheticAddress(self.workdir, 'android-{0}.jar'.format(sdk)) self._jar_library_by_sdk[sdk] = self.context.add_new_target( address, JarLibrary, jars=[jar])
def execute_antlr4_test(self, expected_package): context = self.create_context() task = self.execute(context) # get the synthetic target from the private graph task_outdir = os.path.join(task.workdir, 'antlr4', 'gen-java') syn_sourceroot = os.path.join(task_outdir, self.PARTS['srcroot']) syn_target_name = ('{srcroot}/{dir}.{name}'.format( **self.PARTS)).replace('/', '.') syn_address = SyntheticAddress(spec_path=os.path.relpath( syn_sourceroot, self.build_root), target_name=syn_target_name) syn_target = context.build_graph.get_target(syn_address) # verify that the synthetic target's list of sources match what are actually created def re_relativize(p): """Take a path relative to task_outdir, and make it relative to the build_root""" return os.path.relpath(os.path.join(task_outdir, p), self.build_root) actual_sources = [ re_relativize(s) for s in Fileset.rglobs('*.java', root=task_outdir) ] self.assertEquals(set(syn_target.sources_relative_to_buildroot()), set(actual_sources)) # and that the synthetic target has a valid source root and the generated sources have the # expected java package def get_package(path): with open(path) as fp: for line in fp: match = self.PACKAGE_RE.match(line) if match: return match.group('package_name') return None for source in syn_target.sources_relative_to_source_root(): source_path = os.path.join(syn_sourceroot, source) self.assertTrue( os.path.isfile(source_path), "{0} is not the source root for {1}".format( syn_sourceroot, source)) self.assertEqual(expected_package, get_package(source_path))
def _create_java_target(self, target, dependees): genfiles = [] for source in target.sources_relative_to_source_root(): path = os.path.join(target.target_base, source) genfiles.extend(calculate_genfiles(path, source).get('java', [])) name = '{target_name}'.format(target_name=target.name) spec_path = os.path.relpath(self.java_out, get_buildroot()) spec = '{spec_path}:{name}'.format(spec_path=spec_path, name=name) address = SyntheticAddress(spec=spec) tgt = self.context.add_new_target(address, JavaLibrary, derived_from=target, sources=genfiles, provides=target.provides, dependencies=self.javadeps, excludes=target.payload.excludes) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt
def inject_address_closure_into_build_graph(self, address, build_graph, addresses_already_closed=None): addresses_already_closed = addresses_already_closed or set() if address in addresses_already_closed: return self._populate_target_proxy_transitive_closure_for_address(address) target_proxy = self._target_proxy_by_address[address] if not build_graph.contains_address(address): addresses_already_closed.add(address) for dep_address in target_proxy.dependency_addresses: self.inject_address_closure_into_build_graph( dep_address, build_graph, addresses_already_closed) target = target_proxy.to_target(build_graph) build_graph.inject_target( target, dependencies=target_proxy.dependency_addresses) for traversable_spec in target.traversable_dependency_specs: spec_path, target_name = parse_spec( traversable_spec, relative_to=address.spec_path) self._inject_spec_closure_into_build_graph( spec_path, target_name, build_graph, addresses_already_closed) traversable_spec_target = build_graph.get_target( SyntheticAddress(spec_path, target_name)) if traversable_spec_target not in target.dependencies: build_graph.inject_dependency( dependent=target.address, dependency=traversable_spec_target.address) target.mark_transitive_invalidation_hash_dirty() for traversable_spec in target.traversable_specs: spec_path, target_name = parse_spec( traversable_spec, relative_to=address.spec_path) self._inject_spec_closure_into_build_graph( spec_path, target_name, build_graph, addresses_already_closed) target.mark_transitive_invalidation_hash_dirty()
def create_target(self, binary, gentarget): """Create a JavaLibrary target for the R.java files created by the aapt tool. :param AndroidBinary binary: AndroidBinary target whose target_sdk is used. :param AndroidTarget gentarget: AndroidBinary or Library that owns the processed resources. :returns new_target: Synthetic target for the R.java output of the aapt tool. :rtype::class:`pants.backend.jvm.targets.java_library.JavaLibrary` """ spec_path = os.path.join( os.path.relpath(self.aapt_out(binary), get_buildroot())) address = SyntheticAddress(spec_path=spec_path, target_name=gentarget.id) deps = [self._jar_library_by_sdk[binary.target_sdk]] new_target = self.context.add_new_target( address, JavaLibrary, derived_from=gentarget, sources=[self._relative_genfile(gentarget)], dependencies=deps) return new_target
def _create_java_target(self, target, dependees): genfiles = [] for source in target.sources_relative_to_source_root(): path = os.path.join(target.target_base, source) genfiles.extend(calculate_genfiles( path, source, target.payload.service_writer).get('java', [])) spec_path = os.path.relpath(self.java_out, get_buildroot()) address = SyntheticAddress(spec_path, target.id) deps = OrderedSet(self.javadeps) tgt = self.context.add_new_target(address, JavaLibrary, derived_from=target, sources=genfiles, provides=target.provides, dependencies=deps, excludes=target.payload.excludes) for dependee in dependees: dependee.inject_dependency(tgt.address) return tgt