def _mapped_dependencies(self, jardepmap, binary, confs): # TODO(John Sirois): rework product mapping towards well known types # Generate a map of jars for each unique artifact (org, name) externaljars = OrderedDict() visited = set() for conf in confs: mapped = jardepmap.get((binary, conf)) if mapped: for basedir, jars in mapped.items(): for externaljar in jars: if (basedir, externaljar) not in visited: visited.add((basedir, externaljar)) keys = jardepmap.keys_for(basedir, externaljar) for key in keys: if isinstance(key, tuple) and len(key) == 3: org, name, configuration = key classpath_entry = externaljars.get( (org, name)) if not classpath_entry: classpath_entry = {} externaljars[(org, name)] = classpath_entry classpath_entry[conf] = os.path.join( basedir, externaljar) return externaljars.values()
def _topological_sort(self, goal_info_by_goal): dependees_by_goal = OrderedDict() def add_dependee(goal, dependee=None): dependees = dependees_by_goal.get(goal) if dependees is None: dependees = set() dependees_by_goal[goal] = dependees if dependee: dependees.add(dependee) for goal, goal_info in goal_info_by_goal.items(): add_dependee(goal) for dependency in goal_info.goal_dependencies: add_dependee(dependency, goal) satisfied = set() while dependees_by_goal: count = len(dependees_by_goal) for goal, dependees in dependees_by_goal.items(): unsatisfied = len(dependees - satisfied) if unsatisfied == 0: satisfied.add(goal) dependees_by_goal.pop(goal) yield goal_info_by_goal[goal] break if len(dependees_by_goal) == count: for dependees in dependees_by_goal.values(): dependees.difference_update(satisfied) # TODO(John Sirois): Do a better job here and actually collect and print cycle paths # between Goals/Tasks. The developer can most directly address that data. raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}' .format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees)) for goal, dependees in dependees_by_goal.items())))
def _visit_goal(self, goal, context, goal_info_by_goal): if goal in goal_info_by_goal: return tasks_by_name = OrderedDict() goal_dependencies = set() visited_task_types = set() for task_name in reversed(goal.ordered_task_names()): task_type = goal.task_type_by_name(task_name) visited_task_types.add(task_type) task_workdir = os.path.join( context.options.for_global_scope().pants_workdir, goal.name, task_name) task = task_type(context, task_workdir) tasks_by_name[task_name] = task round_manager = RoundManager(context) task.prepare(round_manager) try: dependencies = round_manager.get_dependencies() for producer_info in dependencies: producer_goal = producer_info.goal if producer_goal == goal: if producer_info.task_type in visited_task_types: ordering = '\n\t'.join( "[{0}] '{1}' {2}".format( i, tn, goal.task_type_by_name(tn).__name__) for i, tn in enumerate( goal.ordered_task_names())) raise self.TaskOrderError( "TaskRegistrar '{name}' with action {consumer_task} depends on {data} from task " "{producer_task} which is ordered after it in the '{goal}' goal:\n\t{ordering}" .format(name=task_name, consumer_task=task_type.__name__, data=producer_info.product_type, producer_task=producer_info.task_type. __name__, goal=goal.name, ordering=ordering)) else: # We don't express dependencies on downstream tasks in this same goal. pass else: goal_dependencies.add(producer_goal) except round_manager.MissingProductError as e: raise self.MissingProductError( "Could not satisfy data dependencies for goal '{name}' with action {action}: {error}" .format(name=task_name, action=task_type.__name__, error=e)) goal_info = self.GoalInfo(goal, tasks_by_name, goal_dependencies) goal_info_by_goal[goal] = goal_info for goal_dependency in goal_dependencies: self._visit_goal(goal_dependency, context, goal_info_by_goal)
def shard_param_docstring(s): """Shard a Target class' sphinx-flavored __init__ docstring by param E.g., if the docstring is :param float x: x coordinate blah blah blah :param y: y coordinate :type y: float should return OrderedDict( 'x' : {'type': 'float', 'param': 'x coordinate\n blah blah blah'}, 'y' : {'type': 'float', 'param': 'y coordinate'}, ) """ # state: what I'm "recording" right now. Needed for multi-line fields. # ('x', 'param') : recording contents of a :param x: blah blah blah # ('x', 'type') : recording contents of a :type x: blah blah blah # ('!forget', '!') not recording useful things; purged before returning state = ('!forget', '!') # shards: return value shards = OrderedDict([('!forget', {'!': ''})]) s = s or '' for line in s.splitlines(): # If this line is indented, keep "recording" whatever we're recording: if line and line[0].isspace(): param, type_or_desc = state shards[param][type_or_desc] += '\n' + line else: # line not indented, starting something new # if a :param foo: line... if param_re.match(line): param_m = param_re.match(line) param_name = param_m.group('param') state = (param_name, 'param') if not param_name in shards: shards[param_name] = {} if param_m.group('type'): shards[param_name]['type'] = param_m.group('type') shards[param_name]['param'] = param_m.group('desc') # if a :type foo: line... elif type_re.match(line): type_m = type_re.match(line) param_name = type_m.group('param') state = (param_name, 'type') if not param_name in shards: shards[param_name] = {} shards[param_name]['type'] = type_m.group('type') # else, nothing that we want to "record" else: state = ('!forget', '!') del shards['!forget'] return shards
def info_for_target_class(cls): """Walk up inheritance tree to get info about constructor args. Helper function for entry_for_one_class. Target classes use inheritance to handle constructor params. If you try to get the argspec for, e.g., `JunitTests.__init__`, it won't mention the `name` parameter, because that's handled by the `Target` superclass. """ # args to not-document. BUILD file authors shouldn't # use these; they're meant to be impl-only. ARGS_SUPPRESS = ['address', 'build_graph', 'payload'] # "accumulate" argspec and docstring fragments going up inheritance tree. suppress = set( ARGS_SUPPRESS) # only show things once. don't show silly things args_accumulator = [] defaults_accumulator = [] docs_accumulator = [] for c in inspect.getmro(cls): if not issubclass(c, Target): continue if not inspect.ismethod(c.__init__): continue args, _, _, defaults = inspect.getargspec(c.__init__) args_that_have_defaults = args[len(args) - len(defaults or ()):] args_with_no_defaults = args[1:(len(args) - len(defaults or ()))] for i in range(len(args_that_have_defaults)): arg = args_that_have_defaults[i] if not arg in suppress: suppress.add(arg) args_accumulator.append(arg) defaults_accumulator.append(defaults[i]) for arg in args_with_no_defaults: if not arg in suppress: suppress.add(arg) args_accumulator.insert(0, arg) dedented_doc = dedent_docstring(c.__init__.__doc__) docs_accumulator.append(shard_param_docstring(dedented_doc)) argspec = inspect.formatargspec(args_accumulator, None, None, defaults_accumulator) suppress = set( ARGS_SUPPRESS) # only show things once. don't show silly things funcdoc_rst = '' funcdoc_shards = OrderedDict() for shard in docs_accumulator: for param, parts in shard.items(): if param in suppress: continue suppress.add(param) funcdoc_shards[param] = parts # Don't interpret param names like "type_" as links. if 'type' in parts: funcdoc_rst += '\n:type {0}: {1}'.format(param, parts['type']) if 'param' in parts: funcdoc_rst += '\n:param {0}: {1}'.format( param, parts['param']) paramdocs = param_docshards_to_template_datas(funcdoc_shards) return (argspec, funcdoc_rst, paramdocs)
def _prepare(self, context, goals): if len(goals) == 0: raise TaskError('No goals to prepare') goal_info_by_goal = OrderedDict() for goal in reversed(OrderedSet(goals)): self._visit_goal(goal, context, goal_info_by_goal) for goal_info in reversed(list(self._topological_sort(goal_info_by_goal))): yield GoalExecutor(context, goal_info.goal, goal_info.tasks_by_name)
def test_execution_minified_dependencies_1(self): dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[]) target_map = self.create_dependencies(dep_map) with self.run_execute(target_map['foo'], recursive=False) as setup_py: setup_py.run_one.assert_called_with(target_map['foo']) with self.run_execute(target_map['foo'], recursive=True) as setup_py: setup_py.run_one.assert_has_calls([ call(target_map['foo']), call(target_map['bar']), call(target_map['baz']) ], any_order=True)
def test_minified_dependencies_2(self): # foo --> baz # | ^ # v | # bar ----' dep_map = OrderedDict(foo=['bar', 'baz'], bar=['baz'], baz=[]) target_map = self.create_dependencies(dep_map) self.assertEqual(SetupPy.minified_dependencies(target_map['foo']), OrderedSet([target_map['bar']])) self.assertEqual(SetupPy.minified_dependencies(target_map['bar']), OrderedSet([target_map['baz']])) self.assertEqual(SetupPy.minified_dependencies(target_map['baz']), OrderedSet())
def __init__(self, timer=None): """Creates a timer that uses time.time for timing intervals by default. :param timer: A callable that returns the current time in fractional seconds. """ self._now = timer or time.time if not(callable(self._now)): # TODO(John Sirois): `def jake(bob): pass` is also callable - we want a no-args callable - # create a better check. raise ValueError('Timer must be a callable object.') self._timings = OrderedDict() self._elapsed = None self._start = self._now()
def _record(self, goal, elapsed): phase = Phase.of(goal) phase_timings = self._timings.get(phase) if phase_timings is None: phase_timings = OrderedDict(()) self._timings[phase] = phase_timings goal_timings = phase_timings.get(goal) if goal_timings is None: goal_timings = [] phase_timings[goal] = goal_timings goal_timings.append(elapsed)
def __init__(self, left_elem, right_elem, keys_only_headers=None): left_type = type(left_elem) right_type = type(right_elem) if left_type != right_type: raise Exception('Cannot compare elements of types %s and %s' % (left_type, right_type)) self._arg_diffs = OrderedDict() for header, left_dict, right_dict in zip(left_elem.headers, left_elem.args, right_elem.args): keys_only = header in (keys_only_headers or []) self._arg_diffs[header] = DictDiff(left_dict, right_dict, keys_only=keys_only)
def test_minified_dependencies_1(self): # foo -> bar -> baz dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[]) target_map = self.create_dependencies(dep_map) self.assertEqual(SetupPy.minified_dependencies(target_map['foo']), OrderedSet([target_map['bar']])) self.assertEqual(SetupPy.minified_dependencies(target_map['bar']), OrderedSet([target_map['baz']])) self.assertEqual(SetupPy.minified_dependencies(target_map['baz']), OrderedSet()) self.assertEqual(SetupPy.install_requires(target_map['foo']), set(['bar==0.0.0'])) self.assertEqual(SetupPy.install_requires(target_map['bar']), set(['baz==0.0.0'])) self.assertEqual(SetupPy.install_requires(target_map['baz']), set([]))
def test_minified_dependencies_diamond(self): # bar <-- foo --> baz # | | # `----> bak <----' dep_map = OrderedDict(foo=['bar', 'baz'], bar=['bak'], baz=['bak'], bak=[]) target_map = self.create_dependencies(dep_map) self.assertEqual(SetupPy.minified_dependencies(target_map['foo']), OrderedSet([target_map['bar'], target_map['baz']])) self.assertEqual(SetupPy.minified_dependencies(target_map['bar']), OrderedSet([target_map['bak']])) self.assertEqual(SetupPy.minified_dependencies(target_map['baz']), OrderedSet([target_map['bak']])) self.assertEqual(SetupPy.install_requires(target_map['foo']), set(['bar==0.0.0', 'baz==0.0.0'])) self.assertEqual(SetupPy.install_requires(target_map['bar']), set(['bak==0.0.0'])) self.assertEqual(SetupPy.install_requires(target_map['baz']), set(['bak==0.0.0']))
for file in files: file = ensure_text(file) full_path = os.path.join(root, file) relpath = os.path.relpath(full_path, basedir) if prefix: relpath = os.path.join(ensure_text(prefix), relpath) zip.write(full_path, relpath) return zippath TAR = TarArchiver('w:', 'tar') TGZ = TarArchiver('w:gz', 'tar.gz') TBZ2 = TarArchiver('w:bz2', 'tar.bz2') ZIP = ZipArchiver(ZIP_DEFLATED) _ARCHIVER_BY_TYPE = OrderedDict(tar=TGZ, tgz=TGZ, tbz2=TBZ2, zip=ZIP) TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys()) def archiver(typename): """Returns Archivers in common configurations. The typename must correspond to one of the following: 'tar' Returns a tar archiver that applies no compression and emits .tar files. 'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files. 'tbz2' Returns a tar archiver that applies bzip2 compression and emits .tar.bz2 files. 'zip' Returns a zip archiver that applies standard compression and emits .zip files. """ archiver = _ARCHIVER_BY_TYPE.get(typename) if not archiver: