def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}'
.format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees))
for goal, dependees
in dependees_by_goal.items())))
def _handle_duplicate_sources(self, vt, sources):
"""Handles duplicate sources generated by the given gen target by either failure or deletion.
This method should be called after all dependencies have been injected into the graph, but
before injecting the synthetic version of this target.
Returns a boolean indicating whether it modified the underlying filesystem.
NB(gm): Some code generators may re-generate code that their dependent libraries generate.
This results in targets claiming to generate sources that they really don't, so we try to
filter out sources that were actually generated by dependencies of the target. This causes
the code generated by the dependencies to 'win' over the code generated by dependees. By
default, this behavior is disabled, and duplication in generated sources will raise a
TaskError. This is controlled by the --allow-dups flag.
"""
target = vt.target
target_workdir = vt.results_dir
# Walk dependency gentargets and record any sources owned by those targets that are also
# owned by this target.
duplicates_by_target = OrderedDict()
def record_duplicates(dep):
if dep == target or not self.is_gentarget(dep.concrete_derived_from):
return False
duped_sources = [s for s in dep.sources_relative_to_source_root() if s in sources.files and
not self.ignore_dup(target, dep, s)]
if duped_sources:
duplicates_by_target[dep] = duped_sources
target.walk(record_duplicates)
# If there were no dupes, we're done.
if not duplicates_by_target:
return False
# If there were duplicates warn or error.
messages = ['{target} generated sources that had already been generated by dependencies.'
.format(target=target.address.spec)]
for dep, duped_sources in duplicates_by_target.items():
messages.append('\t{} also generated:'.format(dep.concrete_derived_from.address.spec))
messages.extend(['\t\t{}'.format(source) for source in duped_sources])
message = '\n'.join(messages)
if self.get_options().allow_dups:
logger.warn(message)
else:
raise self.DuplicateSourceError(message)
did_modify = False
# Finally, remove duplicates from the workdir. This prevents us from having to worry
# about them during future incremental compiles.
for dep, duped_sources in duplicates_by_target.items():
for duped_source in duped_sources:
safe_delete(os.path.join(target_workdir, duped_source))
did_modify = True
if did_modify:
Digest.clear(vt.current_results_dir)
return did_modify
def execute(self):
codegen_targets = self.codegen_targets()
if not codegen_targets:
return
self._validate_sources_globs()
with self.invalidated(
codegen_targets,
invalidate_dependents=True,
topological_order=True,
fingerprint_strategy=self.get_fingerprint_strategy(
)) as invalidation_check:
with self.context.new_workunit(name='execute',
labels=[WorkUnitLabel.MULTITOOL]):
vts_to_sources = OrderedDict()
for vt in invalidation_check.all_vts:
synthetic_target_dir = self.synthetic_target_dir(
vt.target, vt.results_dir)
key = (vt, synthetic_target_dir)
vts_to_sources[key] = None
# Build the target and handle duplicate sources.
if not vt.valid:
if self._do_validate_sources_present(vt.target):
self.execute_codegen(vt.target, vt.results_dir)
sources = self._capture_sources((key, ))[0]
# _handle_duplicate_sources may delete files from the filesystem, so we need to
# re-capture the sources.
if not self._handle_duplicate_sources(
vt.target, vt.results_dir, sources):
vts_to_sources[key] = sources
vt.update()
vts_to_capture = tuple(
key for key, sources in vts_to_sources.items()
if sources is None)
filesets = self._capture_sources(vts_to_capture)
for key, fileset in zip(vts_to_capture, filesets):
vts_to_sources[key] = fileset
for (vt,
synthetic_target_dir), fileset in vts_to_sources.items():
self._inject_synthetic_target(
vt.target,
synthetic_target_dir,
fileset,
)
self._mark_transitive_invalidation_hashes_dirty(
vt.target.address for vt in invalidation_check.all_vts)
def execute(self):
codegen_targets = self.codegen_targets()
if not codegen_targets:
return
self._validate_sources_globs()
with self.invalidated(codegen_targets,
invalidate_dependents=True,
topological_order=True,
fingerprint_strategy=self.get_fingerprint_strategy()) as invalidation_check:
with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]):
vts_to_sources = OrderedDict()
for vt in invalidation_check.all_vts:
vts_to_sources[vt] = None
# Build the target and handle duplicate sources.
if not vt.valid:
if self._do_validate_sources_present(vt.target):
self.execute_codegen(vt.target, vt.current_results_dir)
sources = self._capture_sources((vt,))[0]
# _handle_duplicate_sources may delete files from the filesystem, so we need to
# re-capture the sources.
if not self._handle_duplicate_sources(vt, sources):
vts_to_sources[vt] = sources
vt.update()
vts_to_capture = tuple(key for key, sources in vts_to_sources.items() if sources is None)
filesets = self._capture_sources(vts_to_capture)
for key, fileset in zip(vts_to_capture, filesets):
vts_to_sources[key] = fileset
for vt, fileset in vts_to_sources.items():
self._inject_synthetic_target(vt, fileset)
self._mark_transitive_invalidation_hashes_dirty(
vt.target.address for vt in invalidation_check.all_vts
)
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError(
'Cycle detected in goal dependencies:\n\t{0}'.format(
'\n\t'.join(
'{0} <- {1}'.format(goal, list(dependees))
for goal, dependees in dependees_by_goal.items())))
def _get_python_thrift_library_sources(self, py_thrift_targets):
"""Get file contents for python thrift library targets."""
target_snapshots = OrderedDict(
(t, t.sources_snapshot(
scheduler=self.context._scheduler).directory_digest)
for t in py_thrift_targets)
filescontent_by_target = OrderedDict(
zip(
target_snapshots.keys(),
self.context._scheduler.product_request(
FilesContent, target_snapshots.values())))
thrift_file_sources_by_target = OrderedDict(
(t, [(file_content.path, file_content.content)
for file_content in all_content.dependencies])
for t, all_content in filescontent_by_target.items())
return thrift_file_sources_by_target
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED, 'zip')
_ARCHIVER_BY_TYPE = OrderedDict(tar=TAR, tgz=TGZ, tbz2=TBZ2, zip=ZIP)
archive_extensions = {
name: archiver.extension
for name, archiver in _ARCHIVER_BY_TYPE.items()
}
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset({'zip'})
TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS
def create_archiver(typename):
"""Returns Archivers in common configurations.
:API: public
The typename must correspond to one of the following:
'tar' Returns a tar archiver that applies no compression and emits .tar files.
'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED, 'zip')
_ARCHIVER_BY_TYPE = OrderedDict(
tar=TAR,
tgz=TGZ,
tbz2=TBZ2,
zip=ZIP)
archive_extensions = {
name: archiver.extension for name, archiver in _ARCHIVER_BY_TYPE.items()
}
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset({'zip'})
TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS
def create_archiver(typename):
"""Returns Archivers in common configurations.
:API: public
The typename must correspond to one of the following:
'tar' Returns a tar archiver that applies no compression and emits .tar files.
'tgz' Returns a tar archiver that applies gzip compression and emits .tar.gz files.