class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] # TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here # and in register_options(). known_scopes = ['', 'run-tracker'] for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() self.run_tracker = RunTracker.from_options(self.options) report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() self._expand_goals_and_specs() @property def spec_excludes(self): # Note: Only call after register_options() has been called. return self.options.for_global_scope().spec_excludes @property def global_options(self): return self.options.for_global_scope() def register_options(self): # Add a 'bootstrap' attribute to the register function, so that register_global can # access the bootstrap option values. def register_global(*args, **kwargs): return self.options.register_global(*args, **kwargs) register_global.bootstrap = self.options.bootstrap_option_values() register_global_options(register_global) # This is the first case we have of non-task, non-global options. # The current implementation special-cases RunTracker, and is temporary. # In the near future it will be replaced with a 'Subsystem' abstraction. # But for now this is useful for kicking the tires. def register_run_tracker(*args, **kwargs): self.options.register('run-tracker', *args, **kwargs) RunTracker.register_options(register_run_tracker) for goal in Goal.all(): goal.register_options(self.options) def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append(self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunTask.killall() return result def _do_run(self): # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context( config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.spec_excludes ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals) def _setup_logging(self, global_options): # NB: quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if global_options.quiet else global_options.level.upper() setup_logging(level, log_dir=global_options.logdir)
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Add any extra paths to python path (eg for loading extra source backends) sys.path.extend(bootstrap_options.for_global_scope().pythonpath) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes) self.register_options() self.run_tracker = RunTracker.from_config(self.config) report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() self._expand_goals_and_specs() def get_spec_excludes(self): # Note: Only call after register_options() has been called. return [os.path.join(self.root_dir, spec_exclude) for spec_exclude in self.options.for_global_scope().spec_excludes] @property def global_options(self): return self.options.for_global_scope() def register_options(self): # Add a 'bootstrap' attribute to the register function, so that register_global can # access the bootstrap option values. def register_global(*args, **kwargs): return self.options.register_global(*args, **kwargs) register_global.bootstrap = self.options.bootstrap_option_values() register_global_options(register_global) for goal in Goal.all(): goal.register_options(self.options) def _expand_goals_and_specs(self): logger = logging.getLogger(__name__) goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.is_help: self.options.print_help(goals=goals) sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.get_spec_excludes(), exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append(self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunTask.killall(log.info) return result def _do_run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # TODO(Eric Ayers) We are missing log messages. Set the log level earlier # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context( config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes() ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. bootstrap_options = options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends( plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] # TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here # and in register_options(). known_scopes = ['', 'run-tracker'] for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options( known_scopes=known_scopes) self.register_options() # TODO(Eric Ayers) We are missing log messages. Set the log level earlier # Enable standard python logging for code with no handle to a context/work-unit. self._setup_logging() # NB: self.options are needed for this call. self.run_tracker = RunTracker.from_options(self.options) report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: self.run_tracker.log( Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser( build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() self._expand_goals_and_specs() @property def spec_excludes(self): # Note: Only call after register_options() has been called. return self.options.for_global_scope().spec_excludes @property def global_options(self): return self.options.for_global_scope() def register_options(self): # Add a 'bootstrap' attribute to the register function, so that register_global can # access the bootstrap option values. def register_global(*args, **kwargs): return self.options.register_global(*args, **kwargs) register_global.bootstrap = self.options.bootstrap_option_values() register_global_options(register_global) # This is the first case we have of non-task, non-global options. # The current implementation special-cases RunTracker, and is temporary. # In the near future it will be replaced with a 'Subsystem' abstraction. # But for now this is useful for kicking the tires. def register_run_tracker(*args, **kwargs): self.options.register('run-tracker', *args, **kwargs) RunTracker.register_options(register_run_tracker) for goal in Goal.all(): goal.register_options(self.options) def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning( " Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}." .format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser( self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options. exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses( spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append( self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunTask.killall() return result def _do_run(self): # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) context = Context(config=self.config, options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.spec_excludes) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals) def _setup_logging(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # TODO(John Sirois): Support logging.config.fileConfig so a site can setup fine-grained # logging control and we don't need to be the middleman plumbing an option for each python # standard logging knob. # NB: quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if self.global_options.quiet else self.global_options.level.upper( ) logging_config = { 'version': 1, # required and there is only a version 1 format so far. 'disable_existing_loggers': False } formatters_config = {'brief': {'format': '%(levelname)s] %(message)s'}} handlers_config = { 'console': { 'class': 'logging.StreamHandler', 'formatter': 'brief', # defined above 'level': level } } log_dir = self.global_options.logdir if log_dir: safe_mkdir(log_dir) # This is close to but not quite glog format. Namely the leading levelname is not a single # character and the fractional second is only to millis precision and not micros. glog_date_format = '%m%d %H:%M:%S' glog_format = ( '%(levelname)s %(asctime)s.%(msecs)d %(process)d %(filename)s:%(lineno)d] ' '%(message)s') formatters_config['glog'] = { 'format': glog_format, 'datefmt': glog_date_format } handlers_config['file'] = { 'class': 'logging.handlers.RotatingFileHandler', 'formatter': 'glog', # defined above 'level': level, 'filename': os.path.join(log_dir, 'pants.log'), 'maxBytes': 10 * 1024 * 1024, 'backupCount': 4 } logging_config['formatters'] = formatters_config logging_config['handlers'] = handlers_config logging_config['root'] = { 'level': level, 'handlers': handlers_config.keys() } logging.config.dictConfig(logging_config)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" @classmethod def setUpClass(cls): """Ensure that all code has a config to read from the cache. TODO: Yuck. Get rid of this after plumbing options through in the right places. """ super(BaseTest, cls).setUpClass() Config.cache(Config.load()) def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': os.path.join(self.build_root, '.pants.d'), 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'cache_key_gen_version': '0-test' } BuildRoot().path = self.build_root self.create_file('pants.ini') build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def config(self, overrides=''): """Returns a config valid for the test build root.""" ini_file = os.path.join(get_buildroot(), 'pants.ini') if overrides: with temporary_file(cleanup=False) as fp: fp.write(overrides) fp.close() return Config.load([ini_file, fp.name]) else: return Config.load([ini_file]) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, config='', options=None, target_roots=None, **kwargs): for_task_types = for_task_types or [] options = options or {} new_option_values = defaultdict(dict) # Get values for all new-style options registered by the tasks in for_task_types. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. def register(*rargs, **rkwargs): scoped_options = new_option_values[scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default task_type.register_options(register) # Now override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): new_option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): new_option_values[scope][key] = val return create_context(config=self.config(overrides=config), options=new_option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, **kwargs) def tearDown(self): BuildRoot().reset() SourceRoot.reset() safe_rmtree(self.build_root) BuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, ['none'] will be used. """ compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets( context.targets(), [('default', entry) for entry in classpath or ['none']]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" @classmethod def setUpClass(cls): """Ensure that all code has a config to read from the cache. TODO: Yuck. Get rid of this after plumbing options through in the right places. """ super(BaseTest, cls).setUpClass() Config.cache(Config.load()) def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources = None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.create_file('pants.ini') build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_global_subsystems = set() # Get default values for all options registered by the tasks in for_task_types. # TODO: This is clunky and somewhat repetitive of the real registration code. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default # TODO: Set register.bootstrap here, for good measure? register.scope = on_scope return register task_type.register_options(register_func(scope)) for subsystem in task_type.global_subsystems(): if subsystem not in registered_global_subsystems: subsystem.register_options(register_func(subsystem.qualify_scope(Options.GLOBAL_SCOPE))) registered_global_subsystems.add(subsystem) for subsystem in task_type.task_subsystems(): subsystem.register_options(register_func(subsystem.qualify_scope(scope))) # Now override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(option_values.keys()): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): BuildRoot().reset() SourceRoot.reset() safe_rmtree(self.build_root) BuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, ['none'] will be used. """ compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath or ['none']]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError('You must set a scope on your subsystem type before using it in tests.') optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update(Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp ) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting(self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError( 'You must set a scope on your subsystem type before using it in tests.' ) optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) optionables.add(task_type) extra_scopes.update( [si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([ dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter() ]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables( optionables, extra_scopes=extra_scopes, options=options) context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir @property def subsystems(self): # Subsystems used outside of any task. return SourceRootBootstrapper, Reporting, RunTracker def setup(self): options_bootstrapper = OptionsBootstrapper() bootstrap_options = options_bootstrapper.get_bootstrap_options() # Get logging setup prior to loading backends so that they can log as needed. self._setup_logging(bootstrap_options.for_global_scope()) # Add any extra paths to python path (eg for loading extra source backends) for path in bootstrap_options.for_global_scope().pythonpath: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. plugins = bootstrap_options.for_global_scope().plugins backend_packages = bootstrap_options.for_global_scope().backend_packages build_configuration = load_plugins_and_backends(plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scope_infos = [ScopeInfo.for_global_scope()] # Add scopes for all needed subsystems. subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems()) for subsystem in subsystems: known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM)) # Add scopes for all tasks in all goals. for goal in Goal.all(): known_scope_infos.extend(filter(None, goal.known_scope_infos())) # Now that we have the known scopes we can get the full options. self.options = options_bootstrapper.get_full_options(known_scope_infos) self.register_options(subsystems) # Make the options values available to all subsystems. Subsystem._options = self.options # Now that we have options we can instantiate subsystems. self.run_tracker = RunTracker.global_instance() self.reporting = Reporting.global_instance() report = self.reporting.initial_reporting(self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url)) else: self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)') self.build_file_parser = BuildFileParser(build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) rev = self.options.for_global_scope().build_file_rev if rev: ScmBuildFile.set_rev(rev) ScmBuildFile.set_scm(get_scm()) build_file_type = ScmBuildFile else: build_file_type = FilesystemBuildFile self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser) self._expand_goals_and_specs() # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() @property def spec_excludes(self): # Note: Only call after register_options() has been called. return self.options.for_global_scope().spec_excludes @property def global_options(self): return self.options.for_global_scope() def register_options(self, subsystems): # Standalone global options. GlobalOptionsRegistrar.register_options_on_scope(self.options) # Options for subsystems. for subsystem in subsystems: subsystem.register_options_on_scope(self.options) # TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism # be a subsystem? for goal in Goal.all(): # Register task options. goal.register_options(self.options) def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self.global_options.tag, filter_for_tag)) for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) tgt = self.build_graph.get_target(address) if tag_filter(tgt): self.targets.append(tgt) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunProcessGroup().killall() return result def _do_run(self): # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False is_explain = self.global_options.explain if self.reporting.global_instance().get_options().invalidation_report: invalidation_report = InvalidationReport() else: invalidation_report = None self.reporting.update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker, invalidation_report=invalidation_report) context = Context( options=self.options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.spec_excludes, invalidation_report=invalidation_report ) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown))) return 1 engine = RoundEngine() result = engine.execute(context, self.goals) if invalidation_report: invalidation_report.report() return result def _setup_logging(self, global_options): # NB: quiet help says 'Squelches all console output apart from errors'. level = 'ERROR' if global_options.quiet else global_options.level.upper() setup_logging(level, log_dir=global_options.logdir)
class GoalRunner(object): """Lists installed goals or else executes a named goal.""" def __init__(self, root_dir): """ :param root_dir: The root directory of the pants workspace. """ self.root_dir = root_dir def setup(self): options_bootstrapper = OptionsBootstrapper() # Force config into the cache so we (and plugin/backend loading code) can use it. # TODO: Plumb options in explicitly. options_bootstrapper.get_bootstrap_options() self.config = Config.from_cache() # Load plugins and backends. backend_packages = self.config.getlist('backends', 'packages', []) plugins = self.config.getlist('backends', 'plugins', []) build_configuration = load_plugins_and_backends( plugins, backend_packages) # Now that plugins and backends are loaded, we can gather the known scopes. self.targets = [] known_scopes = [''] for goal in Goal.all(): # Note that enclosing scopes will appear before scopes they enclose. known_scopes.extend(filter(None, goal.known_scopes())) # Now that we have the known scopes we can get the full options. self.new_options = options_bootstrapper.get_full_options( known_scopes=known_scopes) self.register_options() self.run_tracker = RunTracker.from_config(self.config) report = initial_reporting(self.config, self.run_tracker) self.run_tracker.start(report) url = self.run_tracker.run_info.get_info('report_url') if url: self.run_tracker.log(Report.INFO, 'See a report at: %s' % url) else: self.run_tracker.log( Report.INFO, '(To run a reporting server: ./pants goal server)') self.build_file_parser = BuildFileParser( build_configuration=build_configuration, root_dir=self.root_dir, run_tracker=self.run_tracker) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(run_tracker=self.run_tracker, address_mapper=self.address_mapper) with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]): # construct base parameters to be filled in for BuildGraph for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]): build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path) # TODO(pl): This is an unfortunate interface leak, but I don't think # in the long run that we should be relying on "bootstrap" BUILD files # that do nothing except modify global state. That type of behavior # (e.g. source roots, goal registration) should instead happen in # project plugins, or specialized configuration files. self.build_file_parser.parse_build_file_family(build_file) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self.run_tracker.run_info.add_scm_info() self._expand_goals_and_specs() def get_spec_excludes(self): # Note: Only call after register_options() has been called. return [ os.path.join(self.root_dir, spec_exclude) for spec_exclude in self.new_options.for_global_scope().spec_excludes ] @property def global_options(self): return self.new_options.for_global_scope() def register_options(self): # Add a 'bootstrap' attribute to the register function, so that register_global can # access the bootstrap option values. def register_global(*args, **kwargs): return self.new_options.register_global(*args, **kwargs) register_global.bootstrap = self.new_options.bootstrap_option_values() register_global_options(register_global) for goal in Goal.all(): goal.register_options(self.new_options) def _expand_goals_and_specs(self): logger = logging.getLogger(__name__) goals = self.new_options.goals specs = self.new_options.target_specs fail_fast = self.new_options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning( " Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}." .format(goal)) if self.new_options.is_help: self.new_options.print_help(goals=goals) sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser( self.root_dir, self.address_mapper, spec_excludes=self.get_spec_excludes()) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses( spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append( self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] def run(self): def fail(): self.run_tracker.set_root_outcome(WorkUnit.FAILURE) kill_nailguns = self.new_options.for_global_scope().kill_nailguns try: result = self._do_run() if result: fail() except KeyboardInterrupt: fail() # On ctrl-c we always kill nailguns, otherwise they might keep running # some heavyweight compilation and gum up the system during a subsequent run. kill_nailguns = True raise except Exception: fail() raise finally: self.run_tracker.end() # Must kill nailguns only after run_tracker.end() is called, otherwise there may still # be pending background work that needs a nailgun. if kill_nailguns: # TODO: This is JVM-specific and really doesn't belong here. # TODO: Make this more selective? Only kill nailguns that affect state? # E.g., checkstyle may not need to be killed. NailgunTask.killall(log.info) return result def _do_run(self): # TODO(John Sirois): Consider moving to straight python logging. The divide between the # context/work-unit logging and standard python logging doesn't buy us anything. # Enable standard python logging for code with no handle to a context/work-unit. if self.global_options.level: LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper()) logdir = self.global_options.logdir or self.config.get( 'goals', 'logdir', default=None) if logdir: safe_mkdir(logdir) LogOptions.set_log_dir(logdir) prev_log_level = None # If quiet, temporarily change stderr log level to kill init's output. if self.global_options.quiet: prev_log_level = LogOptions.loglevel_name( LogOptions.stderr_log_level()) # loglevel_name can fail, so only change level if we were able to get the current one. if prev_log_level is not None: LogOptions.set_stderr_log_level( LogOptions._LOG_LEVEL_NONE_KEY) log.init('goals') if prev_log_level is not None: LogOptions.set_stderr_log_level(prev_log_level) else: log.init() # Update the reporting settings, now that we have flags etc. def is_quiet_task(): for goal in self.goals: if goal.has_task_of_type(QuietTaskMixin): return True return False # Target specs are mapped to the patterns which match them, if any. This variable is a key for # specs which don't match any exclusion regexes. We know it won't already be in the list of # patterns, because the asterisks in its name make it an invalid regex. _UNMATCHED_KEY = '** unmatched **' def targets_by_pattern(targets, patterns): mapping = defaultdict(list) for target in targets: matched_pattern = None for pattern in patterns: if re.search(pattern, target.address.spec) is not None: matched_pattern = pattern break if matched_pattern is None: mapping[_UNMATCHED_KEY].append(target) else: mapping[matched_pattern].append(target) return mapping is_explain = self.global_options.explain update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker) if self.global_options.exclude_target_regexp: excludes = self.global_options.exclude_target_regexp log.debug('excludes:\n {excludes}'.format( excludes='\n '.join(excludes))) by_pattern = targets_by_pattern(self.targets, excludes) self.targets = by_pattern[_UNMATCHED_KEY] # The rest of this if-statement is just for debug logging. log.debug('Targets after excludes: {targets}'.format( targets=', '.join(t.address.spec for t in self.targets))) excluded_count = sum(len(by_pattern[p]) for p in excludes) log.debug('Excluded {count} target{plural}.'.format( count=excluded_count, plural=('s' if excluded_count != 1 else ''))) for pattern in excludes: log.debug('Targets excluded by pattern {pattern}\n {targets}'. format(pattern=pattern, targets='\n '.join( t.address.spec for t in by_pattern[pattern]))) context = Context(config=self.config, new_options=self.new_options, run_tracker=self.run_tracker, target_roots=self.targets, requested_goals=self.requested_goals, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, spec_excludes=self.get_spec_excludes()) unknown = [] for goal in self.goals: if not goal.ordered_task_names(): unknown.append(goal) if unknown: context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown)) return 1 engine = RoundEngine() return engine.execute(context, self.goals)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources=None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None): for_task_types = for_task_types or [] options = options or {} option_values = defaultdict(dict) registered_subsystems = set() bootstrap_option_values = None # We fill these in after registering bootstrap options. # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. # All this does is make the names available in code, with the default values. # Individual tests can then override the option values they care about. def register_func(on_scope): def register(*rargs, **rkwargs): scoped_options = option_values[on_scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default register.bootstrap = bootstrap_option_values register.scope = on_scope return register # TODO: This sequence is a bit repetitive of the real registration sequence. # Register bootstrap options and grab their default values for use in subsequent registration. GlobalOptionsRegistrar.register_bootstrap_options( register_func(Options.GLOBAL_SCOPE)) bootstrap_option_values = create_option_values( copy.copy(option_values[Options.GLOBAL_SCOPE])) # Now register the full global scope options. GlobalOptionsRegistrar.register_options( register_func(Options.GLOBAL_SCOPE)) # Now register task and subsystem options for relevant tasks. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) task_type.register_options(register_func(scope)) for subsystem in (set(task_type.global_subsystems()) | set(task_type.task_subsystems()) | self._build_configuration.subsystems()): if subsystem not in registered_subsystems: subsystem.register_options( register_func(subsystem.options_scope)) registered_subsystems.add(subsystem) # Now default option values override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. for scope, opts in options.items(): for key, val in opts.items(): option_values[scope][key] = val for scope, opts in self.options.items(): for key, val in opts.items(): option_values[scope][key] = val # Make inner scopes inherit option values from their enclosing scopes. all_scopes = set(option_values.keys()) for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes. all_scopes.update( [si.scope for si in task_type.known_scope_infos()]) # Iterating in sorted order guarantees that we see outer scopes before inner scopes, # and therefore only have to inherit from our immediately enclosing scope. for scope in sorted(all_scopes): if scope != Options.GLOBAL_SCOPE: enclosing_scope = scope.rpartition('.')[0] opts = option_values[scope] for key, val in option_values.get(enclosing_scope, {}).items(): if key not in opts: # Inner scope values override the inherited ones. opts[key] = val context = create_context(options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, 'none')] compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode='w') as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode='w') as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp, ) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning( "Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal) ) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name="bootstrap", labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting( self._global_options, self._is_quiet(), self._run_tracker ) context = Context( options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report, ) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner( context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report, )
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith("BUILD"): return relpath else: return os.path.join(relpath, "BUILD") def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents="", mode="wb"): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode="a") cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, **kwargs): """Creates a target and injects it into the test's build graph. :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.base.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from ) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. for traversable_dependency_spec in target.traversable_dependency_specs: traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path) traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address) if not traversable_dependency_target: raise ValueError( "Tests must make targets for traversable dependency specs ahead of them " "being traversed, {} tried to traverse {} which does not exist.".format( target, traversable_dependency_address ) ) if traversable_dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): # NB: In a normal pants deployment, 'target' is an alias for # `pants.backend.core.targets.dependencies.Dependencies`. We avoid that dependency on the core # backend here since the `BaseTest` is used by lower level tests in base and since the # `Dependencies` type itself is nothing more than an alias for Target that carries along a # pydoc for the BUILD dictionary. return BuildFileAliases(targets={"target": Target}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, ".pants.d") safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[""] = { "pants_workdir": self.pants_workdir, "pants_supportdir": os.path.join(self.build_root, "build-support"), "pants_distdir": os.path.join(self.build_root, "dist"), "pants_configdir": os.path.join(self.build_root, "config"), "cache_key_gen_version": "0-test", } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file("pants.ini") self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context( self, for_task_types=None, options=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None, ): optionables = set() extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError("You must set a scope on your subsystem type before using it in tests.") optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError("You must set a scope on your task type before using it in tests.") optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems() ) ) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) context = create_context( options=option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace, ) Subsystem._options = context.options return context def tearDown(self): SourceRoot.reset() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent( """ %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) """ % dict( target_type=target_type, name=name, sources=("sources=%s," % repr(sources) if sources else ""), resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""), java_sources=( "java_sources=[%s]," % ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources"))) if "java_sources" in kwargs else "" ), provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""), dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""), ) ), ) return self.target("%s:%s" % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, "resources", name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_compile_classpath(self, context, classpath=None): """ Helps actual test cases to populate the 'compile_classpath' products data mapping in the context, which holds the classpath value for targets. :param context: The execution context where the products data mapping lives. :param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used. """ classpath = classpath or [os.path.join(self.build_root, "none")] compile_classpaths = context.products.get_data("compile_classpath", lambda: UnionProducts()) compile_classpaths.add_for_targets(context.targets(), [("default", entry) for entry in classpath]) @contextmanager def add_data(self, context_products, data_type, target, *products): make_products = lambda: defaultdict(MultipleRootedProducts) data_by_target = context_products.get_data(data_type, make_products) with temporary_dir() as outdir: def create_product(product): abspath = os.path.join(outdir, product) with safe_open(abspath, mode="w") as fp: fp.write(product) return abspath data_by_target[target].add_abs_paths(outdir, map(create_product, products)) yield temporary_dir @contextmanager def add_products(self, context_products, product_type, target, *products): product_mapping = context_products.get(product_type) with temporary_dir() as outdir: def create_product(product): with safe_open(os.path.join(outdir, product), mode="w") as fp: fp.write(product) return product product_mapping.add(target, outdir, map(create_product, products)) yield temporary_dir
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" @classmethod def setUpClass(cls): """Ensure that all code has a config to read from the cache. TODO: Yuck. Get rid of this after plumbing options through in the right places. """ super(BaseTest, cls).setUpClass() Config.cache(Config.load()) def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, resources = None, derived_from=None, **kwargs): address = SyntheticAddress.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] dependencies.extend(resources or []) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) return target @property def alias_groups(self): return BuildFileAliases.create(targets={'target': Dependencies}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.new_options = defaultdict(dict) # scope -> key-value mapping. self.new_options[''] = { 'pants_workdir': os.path.join(self.build_root, '.pants.d'), 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist') } BuildRoot().path = self.build_root self.create_file('pants.ini') build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def config(self, overrides=''): """Returns a config valid for the test build root.""" ini_file = os.path.join(get_buildroot(), 'pants.ini') if overrides: with temporary_file(cleanup=False) as fp: fp.write(overrides) fp.close() return Config.load([ini_file, fp.name]) else: return Config.load([ini_file]) def set_new_options_for_scope(self, scope, **kwargs): self.new_options[scope].update(kwargs) def context(self, for_task_types=None, config='', options=None, new_options=None, target_roots=None, **kwargs): for_task_types = for_task_types or [] new_options = new_options or {} new_option_values = defaultdict(dict) # Get values for all new-style options registered by the tasks in for_task_types. for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') # We provide our own test-only registration implementation, bypassing argparse. # When testing we set option values directly, so we don't care about cmd-line flags, config, # env vars etc. In fact, for test isolation we explicitly don't want to look at those. def register(*rargs, **rkwargs): scoped_options = new_option_values[scope] default = rkwargs.get('default') if default is None and rkwargs.get('action') == 'append': default = [] for flag_name in rargs: option_name = flag_name.lstrip('-').replace('-', '_') scoped_options[option_name] = default task_type.register_options(register) # Now override with any caller-specified values. # TODO(benjy): Get rid of the new_options arg, and require tests to call set_new_options. for scope, opts in new_options.items(): for key, val in opts.items(): new_option_values[scope][key] = val for scope, opts in self.new_options.items(): for key, val in opts.items(): new_option_values[scope][key] = val return create_context(config=self.config(overrides=config), new_options = new_option_values, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, **kwargs) def tearDown(self): BuildRoot().reset() SourceRoot.reset() safe_rmtree(self.build_root) BuildFile.clear_cache() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = SyntheticAddress.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) @contextmanager def workspace(self, *buildfiles): with temporary_dir() as root_dir: with BuildRoot().temporary(root_dir): with pushd(root_dir): for buildfile in buildfiles: touch(os.path.join(root_dir, buildfile)) yield os.path.realpath(root_dir) def populate_exclusive_groups(self, context, key=None, classpaths=None, target_predicate=None): """ Helps actual test cases to populate the "exclusives_groups" products data mapping in the context, which holds the classpath values for targets. :param context: The execution context where the products data mapping lives. :param key: key for list of classpaths in the "exclusives_groups" data mapping. None is the default value for most common cases. :param classpaths: a list of classpath strings. If not specified, ['none'] will be used. :param target_predicate: filter predicate for the context.targets(). For most common test cases, None value is good enough. """ exclusives_mapping = ExclusivesMapping(context) exclusives_mapping.set_base_classpath_for_group( key or '<none>', [('default', entry) for entry in classpaths or ['none']]) exclusives_mapping._populate_target_maps(context.targets(target_predicate)) context.products.safe_create_data('exclusives_groups', lambda: exclusives_mapping)