def _mapped_dependencies(self, jardepmap, binary, confs):
# TODO(John Sirois): rework product mapping towards well known types
# Generate a map of jars for each unique artifact (org, name)
externaljars = OrderedDict()
visited = set()
for conf in confs:
mapped = jardepmap.get((binary, conf))
if mapped:
for basedir, jars in mapped.items():
for externaljar in jars:
if (basedir, externaljar) not in visited:
visited.add((basedir, externaljar))
keys = jardepmap.keys_for(basedir, externaljar)
for key in keys:
if isinstance(key, tuple) and len(key) == 3:
org, name, configuration = key
classpath_entry = externaljars.get(
(org, name))
if not classpath_entry:
classpath_entry = {}
externaljars[(org,
name)] = classpath_entry
classpath_entry[conf] = os.path.join(
basedir, externaljar)
return externaljars.values()
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}'
.format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees))
for goal, dependees
in dependees_by_goal.items())))
def _visit_goal(self, goal, context, goal_info_by_goal):
if goal in goal_info_by_goal:
return
tasks_by_name = OrderedDict()
goal_dependencies = set()
visited_task_types = set()
for task_name in reversed(goal.ordered_task_names()):
task_type = goal.task_type_by_name(task_name)
visited_task_types.add(task_type)
task_workdir = os.path.join(
context.options.for_global_scope().pants_workdir, goal.name,
task_name)
task = task_type(context, task_workdir)
tasks_by_name[task_name] = task
round_manager = RoundManager(context)
task.prepare(round_manager)
try:
dependencies = round_manager.get_dependencies()
for producer_info in dependencies:
producer_goal = producer_info.goal
if producer_goal == goal:
if producer_info.task_type in visited_task_types:
ordering = '\n\t'.join(
"[{0}] '{1}' {2}".format(
i, tn,
goal.task_type_by_name(tn).__name__)
for i, tn in enumerate(
goal.ordered_task_names()))
raise self.TaskOrderError(
"TaskRegistrar '{name}' with action {consumer_task} depends on {data} from task "
"{producer_task} which is ordered after it in the '{goal}' goal:\n\t{ordering}"
.format(name=task_name,
consumer_task=task_type.__name__,
data=producer_info.product_type,
producer_task=producer_info.task_type.
__name__,
goal=goal.name,
ordering=ordering))
else:
# We don't express dependencies on downstream tasks in this same goal.
pass
else:
goal_dependencies.add(producer_goal)
except round_manager.MissingProductError as e:
raise self.MissingProductError(
"Could not satisfy data dependencies for goal '{name}' with action {action}: {error}"
.format(name=task_name, action=task_type.__name__,
error=e))
goal_info = self.GoalInfo(goal, tasks_by_name, goal_dependencies)
goal_info_by_goal[goal] = goal_info
for goal_dependency in goal_dependencies:
self._visit_goal(goal_dependency, context, goal_info_by_goal)
def shard_param_docstring(s):
"""Shard a Target class' sphinx-flavored __init__ docstring by param
E.g., if the docstring is
:param float x: x coordinate
blah blah blah
:param y: y coordinate
:type y: float
should return
OrderedDict(
'x' : {'type': 'float', 'param': 'x coordinate\n blah blah blah'},
'y' : {'type': 'float', 'param': 'y coordinate'},
)
"""
# state: what I'm "recording" right now. Needed for multi-line fields.
# ('x', 'param') : recording contents of a :param x: blah blah blah
# ('x', 'type') : recording contents of a :type x: blah blah blah
# ('!forget', '!') not recording useful things; purged before returning
state = ('!forget', '!')
# shards: return value
shards = OrderedDict([('!forget', {'!': ''})])
s = s or ''
for line in s.splitlines():
# If this line is indented, keep "recording" whatever we're recording:
if line and line[0].isspace():
param, type_or_desc = state
shards[param][type_or_desc] += '\n' + line
else: # line not indented, starting something new
# if a :param foo: line...
if param_re.match(line):
param_m = param_re.match(line)
param_name = param_m.group('param')
state = (param_name, 'param')
if not param_name in shards:
shards[param_name] = {}
if param_m.group('type'):
shards[param_name]['type'] = param_m.group('type')
shards[param_name]['param'] = param_m.group('desc')
# if a :type foo: line...
elif type_re.match(line):
type_m = type_re.match(line)
param_name = type_m.group('param')
state = (param_name, 'type')
if not param_name in shards:
shards[param_name] = {}
shards[param_name]['type'] = type_m.group('type')
# else, nothing that we want to "record"
else:
state = ('!forget', '!')
del shards['!forget']
return shards
def info_for_target_class(cls):
"""Walk up inheritance tree to get info about constructor args.
Helper function for entry_for_one_class. Target classes use inheritance
to handle constructor params. If you try to get the argspec for, e.g.,
`JunitTests.__init__`, it won't mention the `name` parameter, because
that's handled by the `Target` superclass.
"""
# args to not-document. BUILD file authors shouldn't
# use these; they're meant to be impl-only.
ARGS_SUPPRESS = ['address', 'build_graph', 'payload']
# "accumulate" argspec and docstring fragments going up inheritance tree.
suppress = set(
ARGS_SUPPRESS) # only show things once. don't show silly things
args_accumulator = []
defaults_accumulator = []
docs_accumulator = []
for c in inspect.getmro(cls):
if not issubclass(c, Target): continue
if not inspect.ismethod(c.__init__): continue
args, _, _, defaults = inspect.getargspec(c.__init__)
args_that_have_defaults = args[len(args) - len(defaults or ()):]
args_with_no_defaults = args[1:(len(args) - len(defaults or ()))]
for i in range(len(args_that_have_defaults)):
arg = args_that_have_defaults[i]
if not arg in suppress:
suppress.add(arg)
args_accumulator.append(arg)
defaults_accumulator.append(defaults[i])
for arg in args_with_no_defaults:
if not arg in suppress:
suppress.add(arg)
args_accumulator.insert(0, arg)
dedented_doc = dedent_docstring(c.__init__.__doc__)
docs_accumulator.append(shard_param_docstring(dedented_doc))
argspec = inspect.formatargspec(args_accumulator, None, None,
defaults_accumulator)
suppress = set(
ARGS_SUPPRESS) # only show things once. don't show silly things
funcdoc_rst = ''
funcdoc_shards = OrderedDict()
for shard in docs_accumulator:
for param, parts in shard.items():
if param in suppress:
continue
suppress.add(param)
funcdoc_shards[param] = parts
# Don't interpret param names like "type_" as links.
if 'type' in parts:
funcdoc_rst += '\n:type {0}: {1}'.format(param, parts['type'])
if 'param' in parts:
funcdoc_rst += '\n:param {0}: {1}'.format(
param, parts['param'])
paramdocs = param_docshards_to_template_datas(funcdoc_shards)
return (argspec, funcdoc_rst, paramdocs)
def _prepare(self, context, goals):
if len(goals) == 0:
raise TaskError('No goals to prepare')
goal_info_by_goal = OrderedDict()
for goal in reversed(OrderedSet(goals)):
self._visit_goal(goal, context, goal_info_by_goal)
for goal_info in reversed(list(self._topological_sort(goal_info_by_goal))):
yield GoalExecutor(context, goal_info.goal, goal_info.tasks_by_name)
def test_execution_minified_dependencies_1(self):
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
with self.run_execute(target_map['foo'], recursive=False) as setup_py:
setup_py.run_one.assert_called_with(target_map['foo'])
with self.run_execute(target_map['foo'], recursive=True) as setup_py:
setup_py.run_one.assert_has_calls([
call(target_map['foo']),
call(target_map['bar']),
call(target_map['baz'])
],
any_order=True)
def test_minified_dependencies_2(self):
# foo --> baz
# | ^
# v |
# bar ----'
dep_map = OrderedDict(foo=['bar', 'baz'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet())
def __init__(self, timer=None):
"""Creates a timer that uses time.time for timing intervals by default.
:param timer: A callable that returns the current time in fractional seconds.
"""
self._now = timer or time.time
if not(callable(self._now)):
# TODO(John Sirois): `def jake(bob): pass` is also callable - we want a no-args callable -
# create a better check.
raise ValueError('Timer must be a callable object.')
self._timings = OrderedDict()
self._elapsed = None
self._start = self._now()
def _record(self, goal, elapsed):
phase = Phase.of(goal)
phase_timings = self._timings.get(phase)
if phase_timings is None:
phase_timings = OrderedDict(())
self._timings[phase] = phase_timings
goal_timings = phase_timings.get(goal)
if goal_timings is None:
goal_timings = []
phase_timings[goal] = goal_timings
goal_timings.append(elapsed)
def __init__(self, left_elem, right_elem, keys_only_headers=None):
left_type = type(left_elem)
right_type = type(right_elem)
if left_type != right_type:
raise Exception('Cannot compare elements of types %s and %s' %
(left_type, right_type))
self._arg_diffs = OrderedDict()
for header, left_dict, right_dict in zip(left_elem.headers,
left_elem.args,
right_elem.args):
keys_only = header in (keys_only_headers or [])
self._arg_diffs[header] = DictDiff(left_dict,
right_dict,
keys_only=keys_only)
def test_minified_dependencies_1(self):
# foo -> bar -> baz
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet())
self.assertEqual(SetupPy.install_requires(target_map['foo']),
set(['bar==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['bar']),
set(['baz==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['baz']), set([]))
def test_minified_dependencies_diamond(self):
# bar <-- foo --> baz
# | |
# `----> bak <----'
dep_map = OrderedDict(foo=['bar', 'baz'],
bar=['bak'],
baz=['bak'],
bak=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(SetupPy.minified_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['bar']),
OrderedSet([target_map['bak']]))
self.assertEqual(SetupPy.minified_dependencies(target_map['baz']),
OrderedSet([target_map['bak']]))
self.assertEqual(SetupPy.install_requires(target_map['foo']),
set(['bar==0.0.0', 'baz==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['bar']),
set(['bak==0.0.0']))
self.assertEqual(SetupPy.install_requires(target_map['baz']),
set(['bak==0.0.0']))
for file in files:
file = ensure_text(file)
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED)
_ARCHIVER_BY_TYPE = OrderedDict(tar=TGZ, tgz=TGZ, tbz2=TBZ2, zip=ZIP)
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
def archiver(typename):
"""Returns Archivers in common configurations.
The typename must correspond to one of the following:
'tar' Returns a tar archiver that applies no compression and emits .tar files.
'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.
'tbz2' Returns a tar archiver that applies bzip2 compression and emits .tar.bz2 files.
'zip' Returns a zip archiver that applies standard compression and emits .zip files.
"""
archiver = _ARCHIVER_BY_TYPE.get(typename)
if not archiver:
