def _mapped_dependencies(self, jardepmap, binary, confs):
# TODO(John Sirois): rework product mapping towards well known types
# Generate a map of jars for each unique artifact (org, name)
externaljars = OrderedDict()
visited = set()
for conf in confs:
mapped = jardepmap.get((binary, conf))
if mapped:
for basedir, jars in mapped.items():
for externaljar in jars:
if (basedir, externaljar) not in visited:
visited.add((basedir, externaljar))
keys = jardepmap.keys_for(basedir, externaljar)
for key in keys:
if isinstance(key, tuple) and len(key) == 3:
org, name, configuration = key
classpath_entry = externaljars.get(
(org, name))
if not classpath_entry:
classpath_entry = {}
externaljars[(org,
name)] = classpath_entry
classpath_entry[conf] = os.path.join(
basedir, externaljar)
return externaljars.values()
class ZincAnalysisElementDiff(object):
def __init__(self, left_elem, right_elem, keys_only_headers=None):
left_type = type(left_elem)
right_type = type(right_elem)
if left_type != right_type:
raise Exception('Cannot compare elements of types %s and %s' % (left_type, right_type))
self._arg_diffs = OrderedDict()
for header, left_dict, right_dict in zip(left_elem.headers, left_elem.args, right_elem.args):
keys_only = header in (keys_only_headers or [])
self._arg_diffs[header] = DictDiff(left_dict, right_dict, keys_only=keys_only)
def is_different(self):
return any([x.is_different() for x in self._arg_diffs.values()])
def __unicode__(self):
parts = []
for header, arg_diff in self._arg_diffs.items():
if arg_diff.is_different():
parts.append('Section "%s" differs:\n' % header)
parts.append(arg_diff)
parts.append('\n\n')
return ''.join(parts) # '' is a unicode, so the entire result will be.
def __str__(self):
if Compatibility.PY3:
return self.__unicode__()
else:
return self.__unicode__().encode('utf-8')
class ZincAnalysisElementDiff(object):
def __init__(self, left_elem, right_elem, keys_only_headers=None):
left_type = type(left_elem)
right_type = type(right_elem)
if left_type != right_type:
raise Exception('Cannot compare elements of types %s and %s' %
(left_type, right_type))
self._arg_diffs = OrderedDict()
for header, left_dict, right_dict in zip(left_elem.headers,
left_elem.args,
right_elem.args):
keys_only = header in (keys_only_headers or [])
self._arg_diffs[header] = DictDiff(left_dict,
right_dict,
keys_only=keys_only)
def is_different(self):
return any([x.is_different() for x in self._arg_diffs.values()])
def __unicode__(self):
parts = []
for header, arg_diff in self._arg_diffs.items():
if arg_diff.is_different():
parts.append('Section "%s" differs:\n' % header)
parts.append(arg_diff)
parts.append('\n\n')
return ''.join(parts) # '' is a unicode, so the entire result will be.
def __str__(self):
if Compatibility.PY3:
return self.__unicode__()
else:
return self.__unicode__().encode('utf-8')
def __init__(self, left_elem, right_elem, keys_only_headers=None):
left_type = type(left_elem)
right_type = type(right_elem)
if left_type != right_type:
raise Exception('Cannot compare elements of types %s and %s' %
(left_type, right_type))
self._arg_diffs = OrderedDict()
for header, left_dict, right_dict in zip(left_elem.headers,
left_elem.args,
right_elem.args):
keys_only = header in (keys_only_headers or [])
self._arg_diffs[header] = DictDiff(left_dict,
right_dict,
keys_only=keys_only)
def _record(self, goal, elapsed):
phase = Phase.of(goal)
phase_timings = self._timings.get(phase)
if phase_timings is None:
phase_timings = OrderedDict(())
self._timings[phase] = phase_timings
goal_timings = phase_timings.get(goal)
if goal_timings is None:
goal_timings = []
phase_timings[goal] = goal_timings
goal_timings.append(elapsed)
def __init__(self, timer=None):
"""Creates a timer that uses time.time for timing intervals by default.
:param timer: A callable that returns the current time in fractional seconds.
"""
self._now = timer or time.time
if not(callable(self._now)):
# TODO(John Sirois): `def jake(bob): pass` is also callable - we want a no-args callable -
# create a better check.
raise ValueError('Timer must be a callable object.')
self._timings = OrderedDict()
self._elapsed = None
self._start = self._now()
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}'
.format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees))
for goal, dependees
in dependees_by_goal.items())))
def _visit_goal(self, goal, context, goal_info_by_goal):
if goal in goal_info_by_goal:
return
tasks_by_name = OrderedDict()
goal_dependencies = set()
visited_task_types = set()
for task_name in reversed(goal.ordered_task_names()):
task_type = goal.task_type_by_name(task_name)
visited_task_types.add(task_type)
task_workdir = os.path.join(
context.options.for_global_scope().pants_workdir, goal.name,
task_name)
task = task_type(context, task_workdir)
tasks_by_name[task_name] = task
round_manager = RoundManager(context)
task.prepare(round_manager)
try:
dependencies = round_manager.get_dependencies()
for producer_info in dependencies:
producer_goal = producer_info.goal
if producer_goal == goal:
if producer_info.task_type in visited_task_types:
ordering = '\n\t'.join(
"[{0}] '{1}' {2}".format(
i, tn,
goal.task_type_by_name(tn).__name__)
for i, tn in enumerate(
goal.ordered_task_names()))
raise self.TaskOrderError(
"TaskRegistrar '{name}' with action {consumer_task} depends on {data} from task "
"{producer_task} which is ordered after it in the '{goal}' goal:\n\t{ordering}"
.format(name=task_name,
consumer_task=task_type.__name__,
data=producer_info.product_type,
producer_task=producer_info.task_type.
__name__,
goal=goal.name,
ordering=ordering))
else:
# We don't express dependencies on downstream tasks in this same goal.
pass
else:
goal_dependencies.add(producer_goal)
except round_manager.MissingProductError as e:
raise self.MissingProductError(
"Could not satisfy data dependencies for goal '{name}' with action {action}: {error}"
.format(name=task_name, action=task_type.__name__,
error=e))
goal_info = self.GoalInfo(goal, tasks_by_name, goal_dependencies)
goal_info_by_goal[goal] = goal_info
for goal_dependency in goal_dependencies:
self._visit_goal(goal_dependency, context, goal_info_by_goal)
def __init__(self, left_elem, right_elem, keys_only_headers=None):
left_type = type(left_elem)
right_type = type(right_elem)
if left_type != right_type:
raise Exception('Cannot compare elements of types %s and %s' % (left_type, right_type))
self._arg_diffs = OrderedDict()
for header, left_dict, right_dict in zip(left_elem.headers, left_elem.args, right_elem.args):
keys_only = header in (keys_only_headers or [])
self._arg_diffs[header] = DictDiff(left_dict, right_dict, keys_only=keys_only)
def shard_param_docstring(s):
"""Shard a Target class' sphinx-flavored __init__ docstring by param
E.g., if the docstring is
:param float x: x coordinate
blah blah blah
:param y: y coordinate
:type y: float
should return
OrderedDict(
'x' : {'type': 'float', 'param': 'x coordinate\n blah blah blah'},
'y' : {'type': 'float', 'param': 'y coordinate'},
)
"""
# state: what I'm "recording" right now. Needed for multi-line fields.
# ('x', 'param') : recording contents of a :param x: blah blah blah
# ('x', 'type') : recording contents of a :type x: blah blah blah
# ('!forget', '!') not recording useful things; purged before returning
state = ('!forget', '!')
# shards: return value
shards = OrderedDict([('!forget', {'!': ''})])
s = s or ''
for line in s.splitlines():
# If this line is indented, keep "recording" whatever we're recording:
if line and line[0].isspace():
param, type_or_desc = state
shards[param][type_or_desc] += '\n' + line
else: # line not indented, starting something new
# if a :param foo: line...
if param_re.match(line):
param_m = param_re.match(line)
param_name = param_m.group('param')
state = (param_name, 'param')
if not param_name in shards:
shards[param_name] = {}
if param_m.group('type'):
shards[param_name]['type'] = param_m.group('type')
shards[param_name]['param'] = param_m.group('desc')
# if a :type foo: line...
elif type_re.match(line):
type_m = type_re.match(line)
param_name = type_m.group('param')
state = (param_name, 'type')
if not param_name in shards:
shards[param_name] = {}
shards[param_name]['type'] = type_m.group('type')
# else, nothing that we want to "record"
else:
state = ('!forget', '!')
del shards['!forget']
return shards
def info_for_target_class(cls):
"""Walk up inheritance tree to get info about constructor args.
Helper function for entry_for_one_class. Target classes use inheritance
to handle constructor params. If you try to get the argspec for, e.g.,
`JunitTests.__init__`, it won't mention the `name` parameter, because
that's handled by the `Target` superclass.
"""
# args to not-document. BUILD file authors shouldn't
# use these; they're meant to be impl-only.
ARGS_SUPPRESS = ['address', 'build_graph', 'payload']
# "accumulate" argspec and docstring fragments going up inheritance tree.
suppress = set(
ARGS_SUPPRESS) # only show things once. don't show silly things
args_accumulator = []
defaults_accumulator = []
docs_accumulator = []
for c in inspect.getmro(cls):
if not issubclass(c, Target): continue
if not inspect.ismethod(c.__init__): continue
args, _, _, defaults = inspect.getargspec(c.__init__)
args_that_have_defaults = args[len(args) - len(defaults or ()):]
args_with_no_defaults = args[1:(len(args) - len(defaults or ()))]
for i in range(len(args_that_have_defaults)):
arg = args_that_have_defaults[i]
if not arg in suppress:
suppress.add(arg)
args_accumulator.append(arg)
defaults_accumulator.append(defaults[i])
for arg in args_with_no_defaults:
if not arg in suppress:
suppress.add(arg)
args_accumulator.insert(0, arg)
dedented_doc = dedent_docstring(c.__init__.__doc__)
docs_accumulator.append(shard_param_docstring(dedented_doc))
argspec = inspect.formatargspec(args_accumulator, None, None,
defaults_accumulator)
suppress = set(
ARGS_SUPPRESS) # only show things once. don't show silly things
funcdoc_rst = ''
funcdoc_shards = OrderedDict()
for shard in docs_accumulator:
for param, parts in shard.items():
if param in suppress:
continue
suppress.add(param)
funcdoc_shards[param] = parts
# Don't interpret param names like "type_" as links.
if 'type' in parts:
funcdoc_rst += '\n:type {0}: {1}'.format(param, parts['type'])
if 'param' in parts:
funcdoc_rst += '\n:param {0}: {1}'.format(
param, parts['param'])
paramdocs = param_docshards_to_template_datas(funcdoc_shards)
return (argspec, funcdoc_rst, paramdocs)
def _prepare(self, context, goals):
if len(goals) == 0:
raise TaskError('No goals to prepare')
goal_info_by_goal = OrderedDict()
for goal in reversed(OrderedSet(goals)):
self._visit_goal(goal, context, goal_info_by_goal)
for goal_info in reversed(list(self._topological_sort(goal_info_by_goal))):
yield GoalExecutor(context, goal_info.goal, goal_info.tasks_by_name)
def test_execution_minified_dependencies_1(self):
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
with self.run_execute(target_map['foo'], recursive=False) as setup_py:
setup_py.run_one.assert_called_with(target_map['foo'])
with self.run_execute(target_map['foo'], recursive=True) as setup_py:
setup_py.run_one.assert_has_calls([
call(target_map['foo']),
call(target_map['bar']),
call(target_map['baz'])
],
any_order=True)
def _topological_sort(self, phase_info_by_phase):
dependees_by_phase = OrderedDict()
def add_dependee(phase, dependee=None):
dependees = dependees_by_phase.get(phase)
if dependees is None:
dependees = set()
dependees_by_phase[phase] = dependees
if dependee:
dependees.add(dependee)
for phase, phase_info in phase_info_by_phase.items():
add_dependee(phase)
for dependency in phase_info.phase_dependencies:
add_dependee(dependency, phase)
satisfied = set()
while dependees_by_phase:
count = len(dependees_by_phase)
for phase, dependees in dependees_by_phase.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(phase)
dependees_by_phase.pop(phase)
yield phase_info_by_phase[phase]
break
if len(dependees_by_phase) == count:
for dependees in dependees_by_phase.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.PhaseCycleError('Cycle detected in phase dependencies:\n\t{0}'
.format('\n\t'.join('{0} <- {1}'.format(phase, list(dependees))
for phase, dependees
in dependees_by_phase.items())))
def test_minified_dependencies_2(self):
# foo --> baz
# | ^
# v |
# bar ----'
dep_map = OrderedDict(foo=['bar', 'baz'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet())
def _mapped_dependencies(self, jardepmap, binary, confs):
# TODO(John Sirois): rework product mapping towards well known types
# Generate a map of jars for each unique artifact (org, name)
externaljars = OrderedDict()
visited = set()
for conf in confs:
mapped = jardepmap.get((binary, conf))
if mapped:
for basedir, jars in mapped.items():
for externaljar in jars:
if (basedir, externaljar) not in visited:
visited.add((basedir, externaljar))
keys = jardepmap.keys_for(basedir, externaljar)
for key in keys:
if isinstance(key, tuple) and len(key) == 3:
org, name, configuration = key
classpath_entry = externaljars.get((org, name))
if not classpath_entry:
classpath_entry = {}
externaljars[(org, name)] = classpath_entry
classpath_entry[conf] = os.path.join(basedir, externaljar)
return externaljars.values()
def test_minified_dependencies_1(self):
# foo -> bar -> baz
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet())
self.assertEqual(SetupPy.install_requires(target_map['foo']),
set(['bar==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['bar']),
set(['baz==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['baz']), set([]))
def test_minified_dependencies_diamond(self):
# bar <-- foo --> baz
# | |
# `----> bak <----'
dep_map = OrderedDict(foo=['bar', 'baz'],
bar=['bak'],
baz=['bak'],
bak=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['bak']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet([target_map['bak']]))
self.assertEqual(SetupPy.install_requires(target_map['foo']),
set(['bar==0.0.0', 'baz==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['bar']),
set(['bak==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['baz']),
set(['bak==0.0.0']))
for file in files:
file = file.decode('utf-8')
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(prefix.decode('utf-8'), relpath)
zip.write(full_path, relpath)
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED)
_ARCHIVER_BY_TYPE = OrderedDict(tar=TGZ, tgz=TGZ, tbz2=TBZ2, zip=ZIP)
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
def archiver(typename):
"""Returns Archivers in common configurations.
The typename must correspond to one of the following:
'tar' Returns a tar archiver that applies no compression and emits .tar files.
'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.
'tbz2' Returns a tar archiver that applies bzip2 compression and emits .tar.bz2 files.
'zip' Returns a zip archiver that applies standard compression and emits .zip files.
"""
archiver = _ARCHIVER_BY_TYPE.get(typename)
if not archiver:
class Timer(object):
"""Provides timing support for goal execution."""
@classmethod
@contextmanager
def begin(cls, timer=None):
"""Begins a new ``Timer`` and yields it in a with context.
The timer will be finished if not already by the block yielded to.
"""
t = Timer(timer)
try:
yield t
finally:
t.finish()
def __init__(self, timer=None):
"""Creates a timer that uses time.time for timing intervals by default.
:param timer: A callable that returns the current time in fractional seconds.
"""
self._now = timer or time.time
if not(callable(self._now)):
# TODO(John Sirois): `def jake(bob): pass` is also callable - we want a no-args callable -
# create a better check.
raise ValueError('Timer must be a callable object.')
self._timings = OrderedDict()
self._elapsed = None
self._start = self._now()
def finish(self):
"""Finishes this timer if not already finished.
Calls to ``timed`` after this will raise a ValueError since the timing window is complete.
"""
if self._elapsed is None:
self._elapsed = self._now() - self._start
@property
def timings(self):
"""Returns the phase timings as an ordered mapping from the ``Phase`` objects executed to
ordered mappings of the ``Goal`` objects executed in the phase to the list of timings
corresponding to each execution of the goal.
Note that the list of timings will be singleton for all goals except those participating in a
``Group``. Grouped goals will have or more timings in the list corresponding to each chunk of
targets the goal executed against when iterating the group.
"""
return self._timings
@property
def elapsed(self):
"""Returns the total elapsed time in fractional seconds from the creation of this timer until
it was ``finished``.
"""
if self._elapsed is None:
raise ValueError('Timer has not been finished yet.')
return self._elapsed
@contextmanager
def timed(self, goal):
"""Records the time taken to execute the yielded block an records this timing against the given
goal's total runtime.
"""
if self._elapsed is not None:
raise ValueError('This timer is already finished.')
start = self._now()
try:
yield
finally:
self._record(goal, self._now() - start)
def _record(self, goal, elapsed):
phase = Phase.of(goal)
phase_timings = self._timings.get(phase)
if phase_timings is None:
phase_timings = OrderedDict(())
self._timings[phase] = phase_timings
goal_timings = phase_timings.get(goal)
if goal_timings is None:
goal_timings = []
phase_timings[goal] = goal_timings
goal_timings.append(elapsed)
def render_timing_report(self):
"""Renders this timer's timings into the classic pants timing report format."""
report = ('Timing report\n'
'=============\n')
for phase, timings in self.timings.items():
phase_time = None
for goal, times in timings.items():
if len(times) > 1:
report += '[%(phase)s:%(goal)s(%(numsteps)d)] %(timings)s -> %(total).3fs\n' % {
'phase': phase.name,
'goal': goal.name,
'numsteps': len(times),
'timings': ','.join('%.3fs' % t for t in times),
'total': sum(times)
}
else:
report += '[%(phase)s:%(goal)s] %(total).3fs\n' % {
'phase': phase.name,
'goal': goal.name,
'total': sum(times)
}
if not phase_time:
phase_time = 0
phase_time += sum(times)
if len(timings) > 1:
report += '[%(phase)s] total: %(total).3fs\n' % {
'phase': phase.name,
'total': phase_time
}
report += 'total: %.3fs' % self.elapsed
return report
for file in files:
file = ensure_text(file)
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED)
_ARCHIVER_BY_TYPE = OrderedDict(tar=TGZ, tgz=TGZ, tbz2=TBZ2, zip=ZIP)
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
def archiver(typename):
"""Returns Archivers in common configurations.
The typename must correspond to one of the following:
'tar' Returns a tar archiver that applies no compression and emits .tar files.
'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.
'tbz2' Returns a tar archiver that applies bzip2 compression and emits .tar.bz2 files.
'zip' Returns a zip archiver that applies standard compression and emits .zip files.
"""
archiver = _ARCHIVER_BY_TYPE.get(typename)
if not archiver:
