class LoaderTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() self.working_set = WorkingSet() for entry in working_set.entries: self.working_set.add_entry(entry) def tearDown(self): Goal.clear() @contextmanager def create_register(self, build_file_aliases=None, register_goals=None, global_subsystems=None, module_name='register'): package_name = b'__test_package_{0}'.format(uuid.uuid4().hex) self.assertFalse(package_name in sys.modules) package_module = types.ModuleType(package_name) sys.modules[package_name] = package_module try: register_module_fqn = b'{0}.{1}'.format(package_name, module_name) register_module = types.ModuleType(register_module_fqn) setattr(package_module, module_name, register_module) sys.modules[register_module_fqn] = register_module def register_entrypoint(function_name, function): if function: setattr(register_module, function_name, function) register_entrypoint('build_file_aliases', build_file_aliases) register_entrypoint('global_subsystems', global_subsystems) register_entrypoint('register_goals', register_goals) yield package_name finally: del sys.modules[package_name] def assert_empty_aliases(self): registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(0, len(registered_aliases.targets)) self.assertEqual(0, len(registered_aliases.objects)) self.assertEqual( 0, len(registered_aliases.context_aware_object_factories)) self.assertEqual(self.build_configuration.subsystems(), set()) def test_load_valid_empty(self): with self.create_register() as backend_package: load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() def test_load_valid_partial_aliases(self): aliases = BuildFileAliases.create(targets={'bob': DummyTarget}, objects={ 'obj1': DummyObject1, 'obj2': DummyObject2 }) with self.create_register( build_file_aliases=lambda: aliases) as backend_package: load_backend(self.build_configuration, backend_package) registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.targets['bob']) self.assertEqual(DummyObject1, registered_aliases.objects['obj1']) self.assertEqual(DummyObject2, registered_aliases.objects['obj2']) self.assertEqual(self.build_configuration.subsystems(), set([DummySubsystem1, DummySubsystem2])) def test_load_valid_partial_goals(self): def register_goals(): Goal.by_name('jack').install(TaskRegistrar('jill', DummyTask)) with self.create_register( register_goals=register_goals) as backend_package: Goal.clear() self.assertEqual(0, len(Goal.all())) load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() self.assertEqual(1, len(Goal.all())) task_names = Goal.by_name('jack').ordered_task_names() self.assertEqual(1, len(task_names)) task_name = task_names[0] self.assertEqual('jill', task_name) def test_load_invalid_entrypoint(self): def build_file_aliases(bad_arg): return BuildFileAliases.create() with self.create_register( build_file_aliases=build_file_aliases) as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_invalid_module(self): with self.create_register(module_name='register2') as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_missing_plugin(self): with self.assertRaises(PluginNotFound): self.load_plugins(['Foobar']) def get_mock_plugin(self, name, version, reg=None, alias=None, after=None): """Make a fake Distribution (optionally with entry points) Note the entry points do not actually point to code in the returned distribution -- the distribution does not even have a location and does not contain any code, just metadata. A module is synthesized on the fly and installed into sys.modules under a random name. If optional entry point callables are provided, those are added as methods to the module and their name (foo/bar/baz in fake module) is added as the requested entry point to the mocked metadata added to the returned dist. :param str name: project_name for distribution (see pkg_resources) :param str version: version for distribution (see pkg_resources) :param callable reg: Optional callable for goal registration entry point :param callable alias: Optional callable for build_file_aliases entry point :param callable after: Optional callable for load_after list entry point """ plugin_pkg = b'demoplugin{0}'.format(uuid.uuid4().hex) pkg = types.ModuleType(plugin_pkg) sys.modules[plugin_pkg] = pkg module_name = b'{0}.{1}'.format(plugin_pkg, 'demo') plugin = types.ModuleType(module_name) setattr(pkg, 'demo', plugin) sys.modules[module_name] = plugin metadata = {} entry_lines = [] if reg is not None: setattr(plugin, 'foo', reg) entry_lines.append('register_goals = {}:foo\n'.format(module_name)) if alias is not None: setattr(plugin, 'bar', alias) entry_lines.append( 'build_file_aliases = {}:bar\n'.format(module_name)) if after is not None: setattr(plugin, 'baz', after) entry_lines.append('load_after = {}:baz\n'.format(module_name)) if entry_lines: entry_data = '[pantsbuild.plugin]\n{}\n'.format( '\n'.join(entry_lines)) metadata = {'entry_points.txt': entry_data} return Distribution(project_name=name, version=version, metadata=MockMetadata(metadata)) def load_plugins(self, plugins): load_plugins(self.build_configuration, plugins, load_from=self.working_set) def test_plugin_load_and_order(self): d1 = self.get_mock_plugin('demo1', '0.0.1', after=lambda: ['demo2']) d2 = self.get_mock_plugin('demo2', '0.0.3') self.working_set.add(d1) # Attempting to load 'demo1' then 'demo2' should fail as 'demo1' requires 'after'=['demo2']. with self.assertRaises(PluginLoadOrderError): self.load_plugins(['demo1', 'demo2']) # Attempting to load 'demo2' first should fail as it is not (yet) installed. with self.assertRaises(PluginNotFound): self.load_plugins(['demo2', 'demo1']) # Installing demo2 and then loading in correct order should work though. self.working_set.add(d2) self.load_plugins(['demo2>=0.0.2', 'demo1']) # But asking for a bad (not installed) version fails. with self.assertRaises(VersionConflict): self.load_plugins(['demo2>=0.0.5']) def test_plugin_installs_goal(self): def reg_goal(): Goal.by_name('plugindemo').install(TaskRegistrar('foo', DummyTask)) self.working_set.add( self.get_mock_plugin('regdemo', '0.0.1', reg=reg_goal)) # Start without the custom goal. self.assertEqual(0, len(Goal.by_name('plugindemo').ordered_task_names())) # Load plugin which registers custom goal. self.load_plugins(['regdemo']) # Now the custom goal exists. self.assertEqual(1, len(Goal.by_name('plugindemo').ordered_task_names())) self.assertEqual('foo', Goal.by_name('plugindemo').ordered_task_names()[0]) def test_plugin_installs_alias(self): def reg_alias(): return BuildFileAliases.create( targets={'pluginalias': DummyTarget}, objects={ 'FROMPLUGIN1': DummyObject1, 'FROMPLUGIN2': DummyObject2 }) self.working_set.add( self.get_mock_plugin('aliasdemo', '0.0.1', alias=reg_alias)) # Start with no aliases. self.assert_empty_aliases() # Now load the plugin which defines aliases. self.load_plugins(['aliasdemo']) # Aliases now exist. registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.targets['pluginalias']) self.assertEqual(DummyObject1, registered_aliases.objects['FROMPLUGIN1']) self.assertEqual(DummyObject2, registered_aliases.objects['FROMPLUGIN2']) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) def test_subsystems(self): def global_subsystems(): return {DummySubsystem1, DummySubsystem2} with self.create_register( global_subsystems=global_subsystems) as backend_package: load_backend(self.build_configuration, backend_package) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2})
class LoaderTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() self.working_set = WorkingSet() for entry in working_set.entries: self.working_set.add_entry(entry) def tearDown(self): Goal.clear() @contextmanager def create_register( self, build_file_aliases=None, register_goals=None, global_subsystems=None, module_name="register" ): package_name = b"__test_package_{0}".format(uuid.uuid4().hex) self.assertFalse(package_name in sys.modules) package_module = types.ModuleType(package_name) sys.modules[package_name] = package_module try: register_module_fqn = b"{0}.{1}".format(package_name, module_name) register_module = types.ModuleType(register_module_fqn) setattr(package_module, module_name, register_module) sys.modules[register_module_fqn] = register_module def register_entrypoint(function_name, function): if function: setattr(register_module, function_name, function) register_entrypoint("build_file_aliases", build_file_aliases) register_entrypoint("global_subsystems", global_subsystems) register_entrypoint("register_goals", register_goals) yield package_name finally: del sys.modules[package_name] def assert_empty_aliases(self): registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(0, len(registered_aliases.target_types)) self.assertEqual(0, len(registered_aliases.target_macro_factories)) self.assertEqual(0, len(registered_aliases.objects)) self.assertEqual(0, len(registered_aliases.context_aware_object_factories)) self.assertEqual(self.build_configuration.subsystems(), set()) def test_load_valid_empty(self): with self.create_register() as backend_package: load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() def test_load_valid_partial_aliases(self): aliases = BuildFileAliases(targets={"bob": DummyTarget}, objects={"obj1": DummyObject1, "obj2": DummyObject2}) with self.create_register(build_file_aliases=lambda: aliases) as backend_package: load_backend(self.build_configuration, backend_package) registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.target_types["bob"]) self.assertEqual(DummyObject1, registered_aliases.objects["obj1"]) self.assertEqual(DummyObject2, registered_aliases.objects["obj2"]) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) def test_load_valid_partial_goals(self): def register_goals(): Goal.by_name("jack").install(TaskRegistrar("jill", DummyTask)) with self.create_register(register_goals=register_goals) as backend_package: Goal.clear() self.assertEqual(0, len(Goal.all())) load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() self.assertEqual(1, len(Goal.all())) task_names = Goal.by_name("jack").ordered_task_names() self.assertEqual(1, len(task_names)) task_name = task_names[0] self.assertEqual("jill", task_name) def test_load_invalid_entrypoint(self): def build_file_aliases(bad_arg): return BuildFileAliases() with self.create_register(build_file_aliases=build_file_aliases) as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_invalid_module(self): with self.create_register(module_name="register2") as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_missing_plugin(self): with self.assertRaises(PluginNotFound): self.load_plugins(["Foobar"]) def get_mock_plugin(self, name, version, reg=None, alias=None, after=None): """Make a fake Distribution (optionally with entry points) Note the entry points do not actually point to code in the returned distribution -- the distribution does not even have a location and does not contain any code, just metadata. A module is synthesized on the fly and installed into sys.modules under a random name. If optional entry point callables are provided, those are added as methods to the module and their name (foo/bar/baz in fake module) is added as the requested entry point to the mocked metadata added to the returned dist. :param string name: project_name for distribution (see pkg_resources) :param string version: version for distribution (see pkg_resources) :param callable reg: Optional callable for goal registration entry point :param callable alias: Optional callable for build_file_aliases entry point :param callable after: Optional callable for load_after list entry point """ plugin_pkg = b"demoplugin{0}".format(uuid.uuid4().hex) pkg = types.ModuleType(plugin_pkg) sys.modules[plugin_pkg] = pkg module_name = b"{0}.{1}".format(plugin_pkg, "demo") plugin = types.ModuleType(module_name) setattr(pkg, "demo", plugin) sys.modules[module_name] = plugin metadata = {} entry_lines = [] if reg is not None: setattr(plugin, "foo", reg) entry_lines.append("register_goals = {}:foo\n".format(module_name)) if alias is not None: setattr(plugin, "bar", alias) entry_lines.append("build_file_aliases = {}:bar\n".format(module_name)) if after is not None: setattr(plugin, "baz", after) entry_lines.append("load_after = {}:baz\n".format(module_name)) if entry_lines: entry_data = "[pantsbuild.plugin]\n{}\n".format("\n".join(entry_lines)) metadata = {"entry_points.txt": entry_data} return Distribution(project_name=name, version=version, metadata=MockMetadata(metadata)) def load_plugins(self, plugins): load_plugins(self.build_configuration, plugins, self.working_set) def test_plugin_load_and_order(self): d1 = self.get_mock_plugin("demo1", "0.0.1", after=lambda: ["demo2"]) d2 = self.get_mock_plugin("demo2", "0.0.3") self.working_set.add(d1) # Attempting to load 'demo1' then 'demo2' should fail as 'demo1' requires 'after'=['demo2']. with self.assertRaises(PluginLoadOrderError): self.load_plugins(["demo1", "demo2"]) # Attempting to load 'demo2' first should fail as it is not (yet) installed. with self.assertRaises(PluginNotFound): self.load_plugins(["demo2", "demo1"]) # Installing demo2 and then loading in correct order should work though. self.working_set.add(d2) self.load_plugins(["demo2>=0.0.2", "demo1"]) # But asking for a bad (not installed) version fails. with self.assertRaises(VersionConflict): self.load_plugins(["demo2>=0.0.5"]) def test_plugin_installs_goal(self): def reg_goal(): Goal.by_name("plugindemo").install(TaskRegistrar("foo", DummyTask)) self.working_set.add(self.get_mock_plugin("regdemo", "0.0.1", reg=reg_goal)) # Start without the custom goal. self.assertEqual(0, len(Goal.by_name("plugindemo").ordered_task_names())) # Load plugin which registers custom goal. self.load_plugins(["regdemo"]) # Now the custom goal exists. self.assertEqual(1, len(Goal.by_name("plugindemo").ordered_task_names())) self.assertEqual("foo", Goal.by_name("plugindemo").ordered_task_names()[0]) def test_plugin_installs_alias(self): def reg_alias(): return BuildFileAliases( targets={"pluginalias": DummyTarget}, objects={"FROMPLUGIN1": DummyObject1, "FROMPLUGIN2": DummyObject2} ) self.working_set.add(self.get_mock_plugin("aliasdemo", "0.0.1", alias=reg_alias)) # Start with no aliases. self.assert_empty_aliases() # Now load the plugin which defines aliases. self.load_plugins(["aliasdemo"]) # Aliases now exist. registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.target_types["pluginalias"]) self.assertEqual(DummyObject1, registered_aliases.objects["FROMPLUGIN1"]) self.assertEqual(DummyObject2, registered_aliases.objects["FROMPLUGIN2"]) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) def test_subsystems(self): def global_subsystems(): return {DummySubsystem1, DummySubsystem2} with self.create_register(global_subsystems=global_subsystems) as backend_package: load_backend(self.build_configuration, backend_package) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2})
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError('You must set a scope on your subsystem type before using it in tests.') optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update(Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources = None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. register_bootstrap_options(register_func(Options.GLOBAL_SCOPE), self.build_root) bootstrap_option_values = create_option_values(copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the remaining global scope options. register_global_options(register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options(register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update(task_type.known_scopes()) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError( 'You must set a scope on your subsystem type before using it in tests.' ) optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) optionables.add(task_type) extra_scopes.update( [si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([ dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter() ]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables( optionables, extra_scopes=extra_scopes, options=options) context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options( register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options( register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update( [si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith("BUILD"): return relpath else: return os.path.join(relpath, "BUILD") def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents="", mode="wb"): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode="a") cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, **kwargs): """Creates a target and injects it into the test's build graph. :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.base.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from ) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. for traversable_dependency_spec in target.traversable_dependency_specs: traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path) traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address) if not traversable_dependency_target: raise ValueError( "Tests must make targets for traversable dependency specs ahead of them " "being traversed, {} tried to traverse {} which does not exist.".format( target, traversable_dependency_address ) ) if traversable_dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): # NB: In a normal pants deployment, 'target' is an alias for # `pants.backend.core.targets.dependencies.Dependencies`. We avoid that dependency on the core # backend here since the `BaseTest` is used by lower level tests in base and since the # `Dependencies` type itself is nothing more than an alias for Target that carries along a # pydoc for the BUILD dictionary. return BuildFileAliases(targets={"target": Target}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, ".pants.d") safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[""] = { "pants_workdir": self.pants_workdir, "pants_supportdir": os.path.join(self.build_root, "build-support"), "pants_distdir": os.path.join(self.build_root, "dist"), "pants_configdir": os.path.join(self.build_root, "config"), "cache_key_gen_version": "0-test", } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file("pants.ini") self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context( self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None, ): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError("You must set a scope on your subsystem type before using it in tests.") optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError("You must set a scope on your task type before using it in tests.") optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems() ) ) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) context = create_context( options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace, ) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent( """ %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) """ % dict( target_type=target_type, name=name, sources=("sources=%s," % repr(sources) if sources else ""), resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""), java_sources=( "java_sources=[%s]," % ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources"))) if "java_sources" in kwargs else "" ), provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""), dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""), ) ), ) return self.target("%s:%s" % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, "resources", name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, "none")] compile_classpaths = context.products.get_data("compile_classpath", lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [("default", entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode="w") as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode="w") as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir