def _topological_sort(self, goal_info_by_goal): dependees_by_goal = OrderedDict() def add_dependee(goal, dependee=None): dependees = dependees_by_goal.get(goal) if dependees is None: dependees = set() dependees_by_goal[goal] = dependees if dependee: dependees.add(dependee) for goal, goal_info in goal_info_by_goal.items(): add_dependee(goal) for dependency in goal_info.goal_dependencies: add_dependee(dependency, goal) satisfied = set() while dependees_by_goal: count = len(dependees_by_goal) for goal, dependees in dependees_by_goal.items(): unsatisfied = len(dependees - satisfied) if unsatisfied == 0: satisfied.add(goal) dependees_by_goal.pop(goal) yield goal_info_by_goal[goal] break if len(dependees_by_goal) == count: for dependees in dependees_by_goal.values(): dependees.difference_update(satisfied) # TODO(John Sirois): Do a better job here and actually collect and print cycle paths # between Goals/Tasks. The developer can most directly address that data. raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}' .format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees)) for goal, dependees in dependees_by_goal.items())))
def _handle_duplicate_sources(self, vt, sources): """Handles duplicate sources generated by the given gen target by either failure or deletion. This method should be called after all dependencies have been injected into the graph, but before injecting the synthetic version of this target. Returns a boolean indicating whether it modified the underlying filesystem. NB(gm): Some code generators may re-generate code that their dependent libraries generate. This results in targets claiming to generate sources that they really don't, so we try to filter out sources that were actually generated by dependencies of the target. This causes the code generated by the dependencies to 'win' over the code generated by dependees. By default, this behavior is disabled, and duplication in generated sources will raise a TaskError. This is controlled by the --allow-dups flag. """ target = vt.target target_workdir = vt.results_dir # Walk dependency gentargets and record any sources owned by those targets that are also # owned by this target. duplicates_by_target = OrderedDict() def record_duplicates(dep): if dep == target or not self.is_gentarget(dep.concrete_derived_from): return False duped_sources = [s for s in dep.sources_relative_to_source_root() if s in sources.files and not self.ignore_dup(target, dep, s)] if duped_sources: duplicates_by_target[dep] = duped_sources target.walk(record_duplicates) # If there were no dupes, we're done. if not duplicates_by_target: return False # If there were duplicates warn or error. messages = ['{target} generated sources that had already been generated by dependencies.' .format(target=target.address.spec)] for dep, duped_sources in duplicates_by_target.items(): messages.append('\t{} also generated:'.format(dep.concrete_derived_from.address.spec)) messages.extend(['\t\t{}'.format(source) for source in duped_sources]) message = '\n'.join(messages) if self.get_options().allow_dups: logger.warn(message) else: raise self.DuplicateSourceError(message) did_modify = False # Finally, remove duplicates from the workdir. This prevents us from having to worry # about them during future incremental compiles. for dep, duped_sources in duplicates_by_target.items(): for duped_source in duped_sources: safe_delete(os.path.join(target_workdir, duped_source)) did_modify = True if did_modify: Digest.clear(vt.current_results_dir) return did_modify
def execute(self): codegen_targets = self.codegen_targets() if not codegen_targets: return self._validate_sources_globs() with self.invalidated( codegen_targets, invalidate_dependents=True, topological_order=True, fingerprint_strategy=self.get_fingerprint_strategy( )) as invalidation_check: with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]): vts_to_sources = OrderedDict() for vt in invalidation_check.all_vts: synthetic_target_dir = self.synthetic_target_dir( vt.target, vt.results_dir) key = (vt, synthetic_target_dir) vts_to_sources[key] = None # Build the target and handle duplicate sources. if not vt.valid: if self._do_validate_sources_present(vt.target): self.execute_codegen(vt.target, vt.results_dir) sources = self._capture_sources((key, ))[0] # _handle_duplicate_sources may delete files from the filesystem, so we need to # re-capture the sources. if not self._handle_duplicate_sources( vt.target, vt.results_dir, sources): vts_to_sources[key] = sources vt.update() vts_to_capture = tuple( key for key, sources in vts_to_sources.items() if sources is None) filesets = self._capture_sources(vts_to_capture) for key, fileset in zip(vts_to_capture, filesets): vts_to_sources[key] = fileset for (vt, synthetic_target_dir), fileset in vts_to_sources.items(): self._inject_synthetic_target( vt.target, synthetic_target_dir, fileset, ) self._mark_transitive_invalidation_hashes_dirty( vt.target.address for vt in invalidation_check.all_vts)
def execute(self): codegen_targets = self.codegen_targets() if not codegen_targets: return self._validate_sources_globs() with self.invalidated(codegen_targets, invalidate_dependents=True, topological_order=True, fingerprint_strategy=self.get_fingerprint_strategy()) as invalidation_check: with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]): vts_to_sources = OrderedDict() for vt in invalidation_check.all_vts: vts_to_sources[vt] = None # Build the target and handle duplicate sources. if not vt.valid: if self._do_validate_sources_present(vt.target): self.execute_codegen(vt.target, vt.current_results_dir) sources = self._capture_sources((vt,))[0] # _handle_duplicate_sources may delete files from the filesystem, so we need to # re-capture the sources. if not self._handle_duplicate_sources(vt, sources): vts_to_sources[vt] = sources vt.update() vts_to_capture = tuple(key for key, sources in vts_to_sources.items() if sources is None) filesets = self._capture_sources(vts_to_capture) for key, fileset in zip(vts_to_capture, filesets): vts_to_sources[key] = fileset for vt, fileset in vts_to_sources.items(): self._inject_synthetic_target(vt, fileset) self._mark_transitive_invalidation_hashes_dirty( vt.target.address for vt in invalidation_check.all_vts )
def _topological_sort(self, goal_info_by_goal): dependees_by_goal = OrderedDict() def add_dependee(goal, dependee=None): dependees = dependees_by_goal.get(goal) if dependees is None: dependees = set() dependees_by_goal[goal] = dependees if dependee: dependees.add(dependee) for goal, goal_info in goal_info_by_goal.items(): add_dependee(goal) for dependency in goal_info.goal_dependencies: add_dependee(dependency, goal) satisfied = set() while dependees_by_goal: count = len(dependees_by_goal) for goal, dependees in dependees_by_goal.items(): unsatisfied = len(dependees - satisfied) if unsatisfied == 0: satisfied.add(goal) dependees_by_goal.pop(goal) yield goal_info_by_goal[goal] break if len(dependees_by_goal) == count: for dependees in dependees_by_goal.values(): dependees.difference_update(satisfied) # TODO(John Sirois): Do a better job here and actually collect and print cycle paths # between Goals/Tasks. The developer can most directly address that data. raise self.GoalCycleError( 'Cycle detected in goal dependencies:\n\t{0}'.format( '\n\t'.join( '{0} <- {1}'.format(goal, list(dependees)) for goal, dependees in dependees_by_goal.items())))
def _get_python_thrift_library_sources(self, py_thrift_targets): """Get file contents for python thrift library targets.""" target_snapshots = OrderedDict( (t, t.sources_snapshot( scheduler=self.context._scheduler).directory_digest) for t in py_thrift_targets) filescontent_by_target = OrderedDict( zip( target_snapshots.keys(), self.context._scheduler.product_request( FilesContent, target_snapshots.values()))) thrift_file_sources_by_target = OrderedDict( (t, [(file_content.path, file_content.content) for file_content in all_content.dependencies]) for t, all_content in filescontent_by_target.items()) return thrift_file_sources_by_target
if prefix: relpath = os.path.join(ensure_text(prefix), relpath) zip.write(full_path, relpath) return zippath TAR = TarArchiver('w:', 'tar') TGZ = TarArchiver('w:gz', 'tar.gz') TBZ2 = TarArchiver('w:bz2', 'tar.bz2') ZIP = ZipArchiver(ZIP_DEFLATED, 'zip') _ARCHIVER_BY_TYPE = OrderedDict(tar=TAR, tgz=TGZ, tbz2=TBZ2, zip=ZIP) archive_extensions = { name: archiver.extension for name, archiver in _ARCHIVER_BY_TYPE.items() } TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys()) TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset({'zip'}) TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS def create_archiver(typename): """Returns Archivers in common configurations. :API: public The typename must correspond to one of the following: 'tar' Returns a tar archiver that applies no compression and emits .tar files. 'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.
return zippath TAR = TarArchiver('w:', 'tar') TGZ = TarArchiver('w:gz', 'tar.gz') TBZ2 = TarArchiver('w:bz2', 'tar.bz2') ZIP = ZipArchiver(ZIP_DEFLATED, 'zip') _ARCHIVER_BY_TYPE = OrderedDict( tar=TAR, tgz=TGZ, tbz2=TBZ2, zip=ZIP) archive_extensions = { name: archiver.extension for name, archiver in _ARCHIVER_BY_TYPE.items() } TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys()) TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset({'zip'}) TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS def create_archiver(typename): """Returns Archivers in common configurations. :API: public The typename must correspond to one of the following: 'tar' Returns a tar archiver that applies no compression and emits .tar files. 'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.