def test_dependants_direct(self):
a = MockTarget('a')
b = MockTarget('b', a)
c = MockTarget('c', b)
d = MockTarget('d', c, a)
e = MockTarget('e', d)
context = Context(ContextTest.config, options={}, target_roots=[a, b, c, d, e])
dependees = context.dependants(lambda t: t in set([e, c]))
self.assertEquals(set([c]), dependees.pop(d))
self.assertEquals(0, len(dependees))
def test_dependents_direct(self):
a = MockTarget('a')
b = MockTarget('b', [a])
c = MockTarget('c', [b])
d = MockTarget('d', [c, a])
e = MockTarget('e', [d])
context = Context(ContextTest.config, options={}, target_roots=[a, b, c, d, e])
dependees = context.dependents(lambda t: t in set([e, c]))
self.assertEquals(set([c]), dependees.pop(d))
self.assertEquals(0, len(dependees))
def prepare_task(task_type, config=None, args=None, targets=None, **kwargs):
"""Prepares a Task for execution.
task_type: The class of the Task to create.
config: An optional string representing the contents of a pants.ini config.
args: optional list of command line flags, these should be prefixed with '--test-'.
targets: optional list of Target objects passed on the command line.
**kwargs: Any additional args the Task subclass constructor takes beyond the required context.
Returns a new Task ready to execute.
"""
assert issubclass(
task_type,
Task), 'task_type must be a Task subclass, got %s' % task_type
config = create_config(config or '')
parser = OptionParser()
option_group = OptionGroup(parser, 'test')
mkflag = Mkflag('test')
task_type.setup_parser(option_group, args, mkflag)
options, _ = parser.parse_args(args or [])
run_tracker = create_run_tracker()
context = Context(config, options, run_tracker, targets or [])
return task_type(context, **kwargs)
def run(self, lock):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.options.log_level:
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
log.init('goals')
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_console_task():
for phase in self.phases:
for goal in phase.goals():
if issubclass(goal.task_type, ConsoleTask):
return True
return False
is_explain = self.options.explain
update_reporting(self.options, is_console_task() or is_explain, self.run_tracker)
if self.options.dry_run:
print('****** Dry Run ******')
context = Context(
self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
_list_goals(context, 'Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
return 1
return Goal._execute(context, self.phases, print_timing=self.options.time)
def execute(cls, context, *names):
parser = OptionParser()
cls.add_global_options(parser)
phases = [Phase(name) for name in names]
Phase.setup_parser(parser, [], phases)
options, _ = parser.parse_args([])
context = Context(context.config, options, context.run_tracker, context.target_roots,
requested_goals=list(names))
return cls._execute(context, phases, print_timing=False)
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
def is_console_task():
for phase in self.phases:
for goal in phase.goals():
if issubclass(goal.task_type, ConsoleTask):
return True
return False
update_reporting(self.options, is_console_task(), self.run_tracker)
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def run(self, lock):
with self.check_errors("Target contains a dependency cycle") as error:
with self.timer.timing('parse:check_cycles'):
for target in self.targets:
try:
InternalTarget.check_cycles(target)
except InternalTarget.CycleException as e:
error(target.id)
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config, self.options, self.targets, lock=lock, log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret
def testPartitioning(self):
# Target e has conflicts; in this test, we want to check that partitioning
# of valid targets works to prevent conflicts in chunks, so we only use a-d.
a, b, c, d, _ = self.setupTargets()
context = Context(ExclusivesTargetTest.config, options={}, run_tracker=None, target_roots=[a, b, c, d])
context.products.require_data('exclusives_groups')
check_exclusives_task = CheckExclusives(context, signal_error=True)
check_exclusives_task.execute([a, b, c, d])
egroups = context.products.get_data('exclusives_groups')
self.assertEquals(egroups.get_targets_for_group_key("a=1"), set([a, b, d]))
self.assertEquals(egroups.get_targets_for_group_key("a=2"), set([c]))
def test_check_exclusives(self):
a, b, c, d, e = self.setupTargets()
context = Context(CheckExclusivesTest.config,
options={},
run_tracker=None,
target_roots=[d, e])
check_exclusives_task = CheckExclusives(context, signal_error=True)
try:
check_exclusives_task.execute([d, e])
self.fail(
"Expected a conflicting exclusives exception to be thrown.")
except TaskError:
pass
def create_context(config='', options=None, target_roots=None, **kwargs):
"""Creates a ``Context`` with no config values, options, or targets by default.
:param config: Either a ``Context`` object or else a string representing the contents of the
pants.ini to parse the config from.
:param options: An optional dict of of option values.
:param target_roots: An optional list of target roots to seed the context target graph from.
:param ``**kwargs``: Any additional keyword arguments to pass through to the Context constructor.
"""
config = config if isinstance(config, Config) else create_config(config)
run_tracker = create_run_tracker()
target_roots = maybe_list(target_roots, Target) if target_roots else []
return Context(config, create_options(options or {}), run_tracker,
target_roots, **kwargs)
def test_check_exclusives(self):
a = MockTarget('a', exclusives={'a': '1', 'b': '1'})
b = MockTarget('b', exclusives={'a': '1'})
c = MockTarget('c', exclusives={'a': '2'})
d = MockTarget('d', dependencies=[a, b])
e = MockTarget('e', dependencies=[a, c], exclusives={'c': '1'})
context = Context(CheckExclusivesTest.config,
options={},
run_tracker=None,
target_roots=[d, e])
check_exclusives_task = CheckExclusives(context, signal_error=True)
try:
check_exclusives_task.execute([d, e])
self.fail(
"Expected a conflicting exclusives exception to be thrown.")
except TaskError:
pass
def testClasspathUpdates(self):
# Check that exclusive groups classpaths accumulate properly.
a = MockTarget('a', exclusives={'a': '1', 'b': '1'})
b = MockTarget('b', exclusives={'a': '1', 'b': '<none>'})
c = MockTarget('c', exclusives={'a': '2', 'b': '2'})
d = MockTarget('d')
context = Context(CheckExclusivesTest.config,
options={},
run_tracker=None,
target_roots=[a, b, c, d])
context.products.require_data('exclusives_groups')
check_exclusives_task = CheckExclusives(context, signal_error=True)
check_exclusives_task.execute([a, b, c, d])
egroups = context.products.get_data('exclusives_groups')
egroups.set_base_classpath_for_group("a=1,b=1", ["a1", "b1"])
egroups.set_base_classpath_for_group("a=1,b=<none>", ["a1"])
egroups.set_base_classpath_for_group("a=2,b=2", ["a2", "b2"])
egroups.set_base_classpath_for_group("a=<none>,b=<none>", ["none"])
egroups.update_compatible_classpaths(None, ["update_without_group"])
egroups.update_compatible_classpaths("a=<none>,b=<none>",
["update_all"])
egroups.update_compatible_classpaths("a=1,b=<none>", ["update_a1bn"])
egroups.update_compatible_classpaths("a=2,b=2", ["update_only_a2b2"])
self.assertEquals(egroups.get_classpath_for_group("a=2,b=2"), [
"a2", "b2", "update_without_group", "update_all",
"update_only_a2b2"
])
self.assertEquals(
egroups.get_classpath_for_group("a=1,b=1"),
["a1", "b1", "update_without_group", "update_all", "update_a1bn"])
self.assertEquals(
egroups.get_classpath_for_group("a=1,b=<none>"),
["a1", "update_without_group", "update_all", "update_a1bn"])
self.assertEquals(egroups.get_classpath_for_group("a=<none>,b=<none>"),
["none", "update_without_group", "update_all"])
# make sure repeated additions of the same thing are idempotent.
egroups.update_compatible_classpaths("a=1,b=1", ["a1", "b1", "xxx"])
self.assertEquals(egroups.get_classpath_for_group("a=1,b=1"), [
"a1", "b1", "update_without_group", "update_all", "update_a1bn",
"xxx"
])
def make_dag(nodes):
return DoubleDag(nodes, lambda t: t.dependencies, Context.Log())
def test_dependents_empty(self):
context = Context(ContextTest.config, options={}, target_roots=[])
dependees = context.dependents()
self.assertEquals(0, len(dependees))
def test_classpath_compatibility(self):
# test the compatibility checks for different exclusive groups.
a = MockTarget('a', exclusives={'a': '1', 'b': '1'})
b = MockTarget('b', exclusives={'a': '1', 'b': '<none>'})
c = MockTarget('c', exclusives={'a': '2', 'b': '2'})
d = MockTarget('d')
context = Context(CheckExclusivesTest.config,
options={},
run_tracker=None,
target_roots=[a, b, c, d])
context.products.require_data('exclusives_groups')
check_exclusives_task = CheckExclusives(context, signal_error=True)
check_exclusives_task.execute([a, b, c, d])
egroups = context.products.get_data('exclusives_groups')
# Expected compatibility:
# a is compatible with nothing but itself.
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[a],
egroups.target_to_key[a]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[a],
egroups.target_to_key[b]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[a],
egroups.target_to_key[d]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[a],
egroups.target_to_key[c]))
# b is compatible with itself and a.
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[b],
egroups.target_to_key[a]))
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[b],
egroups.target_to_key[b]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[b],
egroups.target_to_key[c]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[b],
egroups.target_to_key[d]))
# c is compatible with nothing but itself
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[c],
egroups.target_to_key[c]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[c],
egroups.target_to_key[a]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[c],
egroups.target_to_key[b]))
self.assertFalse(
egroups._is_compatible(egroups.target_to_key[c],
egroups.target_to_key[d]))
# d is compatible with everything.
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[d],
egroups.target_to_key[a]))
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[d],
egroups.target_to_key[b]))
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[d],
egroups.target_to_key[c]))
self.assertTrue(
egroups._is_compatible(egroups.target_to_key[d],
egroups.target_to_key[d]))
def run(self, lock):
timer = None
if self.options.time:
class Timer(object):
def now(self):
return time.time()
def log(self, message):
print(message)
timer = Timer()
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level
or 'info').upper())
logdir = self.options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config,
self.options,
self.targets,
lock=lock,
log=logger)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
return Phase.attempt(context, self.phases, timer=timer)
def run(self, lock):
# Update the reporting settings, now that we have flags etc.
log_level = Report.log_level_from_string(self.options.log_level
or 'info')
color = not self.options.no_color
timing = self.options.time
cache_stats = self.options.time # TODO: Separate flag for this?
settings_updates_map = {
'console': {
'log_level': log_level,
'color': color,
'timing': timing,
'cache_stats': cache_stats
},
'html': {
'log_level': log_level
}
}
self.run_tracker.update_report_settings(settings_updates_map)
# TODO: Do something useful with --logdir.
if self.options.dry_run:
print '****** Dry Run ******'
context = Context(self.config,
self.options,
self.run_tracker,
self.targets,
requested_goals=self.requested_goals,
lock=lock)
# TODO: Time to get rid of this hack.
if self.options.recursive_directory:
context.log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
context.log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
return ret
def create_context(cls, **kwargs):
return Context(cls.config,
run_tracker=None,
target_base=MockTarget,
**kwargs)
def test_dependants_empty(self):
context = Context(ContextTest.config, options={}, target_roots=[])
dependees = context.dependants()
self.assertEquals(0, len(dependees))
def create_context(cls, **kwargs):
return Context(cls.config, run_tracker=None, **kwargs)
def run(self, lock):
if self.options.dry_run:
print '****** Dry Run ******'
logger = None
if self.options.log or self.options.log_level:
from twitter.common.log import init
from twitter.common.log.options import LogOptions
LogOptions.set_stderr_log_level((self.options.log_level
or 'info').upper())
logdir = self.options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
init('goals')
else:
init()
logger = log
if self.options.recursive_directory:
log.warn(
'--all-recursive is deprecated, use a target spec with the form [dir]:: instead'
)
for dir in self.options.recursive_directory:
self.add_target_recursive(dir)
if self.options.target_directory:
log.warn(
'--all is deprecated, use a target spec with the form [dir]: instead'
)
for dir in self.options.target_directory:
self.add_target_directory(dir)
context = Context(self.config,
self.options,
self.targets,
requested_goals=self.requested_goals,
lock=lock,
log=logger,
timer=self.timer if self.options.time else None)
unknown = []
for phase in self.phases:
if not phase.goals():
unknown.append(phase)
if unknown:
print('Unknown goal(s): %s' % ' '.join(phase.name
for phase in unknown))
print('')
return Phase.execute(context, 'goals')
if logger:
logger.debug('Operating on targets: %s', self.targets)
ret = Phase.attempt(context, self.phases)
if self.options.cleanup_nailguns or self.config.get(
'nailgun', 'autokill', default=False):
if log:
log.debug('auto-killing nailguns')
if NailgunTask.killall:
NailgunTask.killall(log)
if self.options.time:
print('Timing report')
print('=============')
self.timer.print_timings()
return ret