def _scala_library_used_addresses(self, target):
"""Consults the analysis products to construct a set of addresses a scala library uses."""
syms = self._symbols_used_by_scala_target(target)
used_addresses = set()
errors = []
for symbol in syms:
exact_matching_sources = self._internal_symbol_tree.get(
symbol, exact=False)
manually_defined_target = check_manually_defined(
symbol, subtree=self.merged_map)
if manually_defined_target and exact_matching_sources:
print(
'ERROR: buildgen found both sources and manually defined in third_party_map_jvm'
' targets for this symbol.\n'
'Target: {0}\n'
'Jvm symbol used by target: {1}\n'
'Manually defined target for symbol: {3}\n'
'Sources found defining symbol: \n{2}\n'.format(
target.address.reference(),
symbol,
'\n'.join(' * {0}'.format(source)
for source in exact_matching_sources),
self._manually_defined_spec_to_address(
manually_defined_target).reference(),
))
errors.append((target.address.reference(), symbol))
continue
elif exact_matching_sources:
addresses = set(
chain.from_iterable(
self._source_mapper.target_addresses_for_source(source)
for source in exact_matching_sources))
elif manually_defined_target == 'SKIP':
continue
elif manually_defined_target:
addresses = [
self._manually_defined_spec_to_address(
manually_defined_target)
]
else:
errors.append((target.address.reference(), symbol))
continue
for address in addresses:
dep = self.context.build_graph.get_target(address)
if not dep:
raise UsedSymbolException(
"An address was used that was not injected into the build graph! Make sure that "
"there is a matching BUILD definition for this used address: {}"
.format(address), )
if address == target.address:
pass
elif self._is_test(dep) and not self._is_test(target):
pass
else:
# In the end, we always actually depend on concrete targets. But for now we preserve
# the information that this dependency (could have been) synthetic, and let downstream
# consumers normalize this to a concrete target if necessary.
used_addresses.add(dep.address)
if errors:
print(self._internal_symbol_tree.render())
print(
'ERROR: Failed to map these symbols used by the following target to a providing'
' target:',
file=sys.stderr)
err_msg = []
for spec, symbol in errors:
err_msg.append("")
err_msg.append("Symbol: " + symbol)
err_msg.append("Target: " + spec)
err_msg.append(
'Failed to map scala libraries to used symbols. See error output above.'
)
raise Exception('\n'.join(err_msg))
return used_addresses
def _scala_library_used_addresses(self, target):
"""Consults the analysis products to construct a set of addresses a scala library uses."""
syms = self._symbols_used_by_scala_target(target)
used_addresses = set()
errors = []
for symbol in syms:
exact_matching_sources = self._internal_symbol_tree.get(symbol, exact=False)
manually_defined_target = check_manually_defined(symbol, subtree=self.merged_map)
if manually_defined_target and exact_matching_sources:
print(
'ERROR: buildgen found both sources and manually defined in third_party_map_jvm'
' targets for this symbol.\n'
'Target: {0}\n'
'Jvm symbol used by target: {1}\n'
'Manually defined target for symbol: {3}\n'
'Sources found defining symbol: \n{2}\n'
.format(
target.address.reference(),
symbol,
'\n'.join(' * {0}'.format(source) for source in exact_matching_sources),
self._manually_defined_spec_to_address(manually_defined_target).reference(),
)
)
errors.append((target.address.reference(), symbol))
continue
elif exact_matching_sources:
addresses = set(chain.from_iterable(
self._source_mapper.target_addresses_for_source(source)
for source in exact_matching_sources
))
elif manually_defined_target == 'SKIP':
continue
elif manually_defined_target:
addresses = [self._manually_defined_spec_to_address(manually_defined_target)]
else:
errors.append((target.address.reference(), symbol))
continue
for address in addresses:
dep = self.context.build_graph.get_target(address)
# NOTE(mateo): This cannot happen when using pom-resolve, but OSS consumers have been bitten when dep is None.
# I was unable to repro using Ivy, but this check is cheap enough to be worth it no matter the resolver.
if not dep:
raise UsedSymbolException("An address was used that was not injected into the build graph! Make sure that "
"there is a matching BUILD definition for this used address: {}".format(address))
if address == target.address:
pass
elif self._is_test(dep) and not self._is_test(target):
pass
else:
# In the end, we always actually depend on concrete targets. But for now we preserve
# the information that this dependency (could have been) synthetic, and let downstream
# consumers normalize this to a concrete target if necessary.
used_addresses.add(dep.address)
if errors:
print('ERROR: Failed to map these symbols used by the following target to a providing'
' target:', file=sys.stderr)
err_msg = []
for spec, symbol in errors:
err_msg.append("")
err_msg.append("Symbol: " + symbol)
err_msg.append("Target: " + spec)
err_msg.append('Failed to map scala libraries to used symbols. See error output above.')
raise Exception('\n'.join(err_msg))
return used_addresses
def buildgen_target(self, target):
safe_mkdir(self.workdir)
source_files = [f for f in target.sources_relative_to_buildroot() if f.endswith('.py')]
build_graph = self.context.build_graph
source_to_addresses_mapper = self.context.products.get_data('source_to_addresses_mapper')
# Gather symbols imported from first party source files.
target_used_symbols = set()
addresses_used_by_target = set()
for source_candidate in source_files:
try:
used_symbols = self.get_used_symbols(source_candidate)
except Exception as e:
raise Exception(
dedent(
"""
{}\n
Source file: {}
Referenced by: {}
""".format(e, source_candidate, target.address.spec)
)
)
target_used_symbols.update(used_symbols)
for symbol in target_used_symbols:
prefix = symbol.split('.')[0]
# If the symbol is a first party package, map it to providing source files and memoize the relation.
if prefix in self.first_party_packages:
if symbol not in _SYMBOLS_TO_SOURCES_MAP:
providing_sources = self.symbol_to_source_tree.get(symbol, allow_prefix_imports=True)
if not providing_sources:
raise Exception(
'While python buildgenning {}, encountered a symbol with'
' no providing target. This probably means the import moved'
' or is misspelled. It could also mean that there is no BUILD'
' target that owns the source that provides the symbol.'
' Imported symbol: {}'
.format(target.address.spec, symbol)
)
_SYMBOLS_TO_SOURCES_MAP[symbol] = providing_sources
# Map source file to concrete addresses, tracing codegen back to its concrete target, and cache it.
for source in _SYMBOLS_TO_SOURCES_MAP[symbol]:
if source not in _SOURCE_FILE_TO_ADDRESS_MAP:
target_addresses = set()
for address in source_to_addresses_mapper.target_addresses_for_source(source):
concrete = build_graph.get_concrete_derived_from(address)
if concrete.type_alias != 'python_binary':
target_addresses.add(concrete.address)
_SOURCE_FILE_TO_ADDRESS_MAP[source].update(target_addresses)
addresses_used_by_target.update(_SOURCE_FILE_TO_ADDRESS_MAP[source])
# Since the symbol is not first party, it needs to be mapped to a target.
else:
# The twitter.commons/apache.aurora packages are hopeless to comprehensively map. They have a number of issues:
# * They share common `top_level.txt` and 'namespace_packages.txt' values
# * Heuristics based off the package name are invalid
# * apache.aurora.thrift == gen.apache.aurora.[modules].{1.py, 2.py, 3.py}
# * apache.aurora.thermos == gen.apache.thermos.{a.py, b.py, c.py}
# * how to tell that gen.apache.thermos is not a module of gen.apache.aurora?
# * Consequently:
# * they clobber each other's namespace
# * There is no programmatic way to tell between modules
#
# The valid imports are trivial to determine but there is no deterministic way to map that to the package name.
# Without these, we could entirely rely on the virtualenv introspection but instead third_party_map lives.
if not self.opt_out_virtualenv_walk:
import_map = self.symbol_to_target_map
if prefix not in import_map:
# Slice the import ever shorter until we either match to a known import or run out of parts.
prefix = symbol
parts = len(prefix.split('.'))
while prefix not in import_map and parts > 1:
prefix, _ = prefix.rsplit('.', 1)
parts -= 1
else:
import_map = {}
# Both of these return a target spec string if there is a match and None otherwise.
dep = check_manually_defined(symbol, self.get_options().third_party_map) or import_map.get(prefix)
if not dep:
msg = dedent(
"""\
While running python buildgen, a symbol was found without a known providing target.
Target: {}
Symbol: {}
""".format(target.address.spec, symbol)
)
# TODO(mateo): Make this exception fail-slow. Better to gather all bg failures and print at end.
if self.get_options().fatal:
raise PythonBuildgenError(msg)
else:
print('{}Ignoring for now since fatal errors are off'.format(msg))
else:
addresses_used_by_target.add(Address.parse(dep))
# Remove any imports from within the same module.
filtered_addresses_used_by_target = {
addr for addr in addresses_used_by_target
if addr != target.address
}
self.adjust_target_build_file(target, filtered_addresses_used_by_target)
def buildgen_target(self, target):
safe_mkdir(self.workdir)
source_files = [f for f in target.sources_relative_to_buildroot() if f.endswith('.py')]
build_graph = self.context.build_graph
address_mapper = self.context.address_mapper
source_to_addresses_mapper = self.context.products.get_data('source_to_addresses_mapper')
# Gather symbols imported from first party source files.
target_used_symbols = set()
addresses_used_by_target = set()
for source_candidate in source_files:
target_used_symbols.update(self.get_used_symbols(source_candidate))
for symbol in target_used_symbols:
prefix = symbol.split('.')[0]
# If the symbol is a first party package, map it to providing source files and memoize the relation.
if prefix in self.first_party_packages:
if symbol not in _SYMBOLS_TO_SOURCES_MAP:
providing_sources = self.symbol_to_source_tree.get(symbol, allow_prefix_imports=True)
if not providing_sources:
raise Exception(
'While python buildgenning {}, encountered a symbol with'
' no providing target. This probably means the import moved'
' or is misspelled. It could also mean that there is no BUILD'
' target that owns the source that provides the symbol.'
' Imported symbol: {}'
.format(target.address.spec, symbol)
)
_SYMBOLS_TO_SOURCES_MAP[symbol] = providing_sources
# Map source file to concrete addresses, tracing codegen back to its concrete target, and cache it.
for source in _SYMBOLS_TO_SOURCES_MAP[symbol]:
if source not in _SOURCE_FILE_TO_ADDRESS_MAP:
target_addresses = set()
for address in source_to_addresses_mapper.target_addresses_for_source(source):
concrete = build_graph.get_concrete_derived_from(address)
if concrete.type_alias != 'python_binary':
target_addresses.add(concrete.address)
_SOURCE_FILE_TO_ADDRESS_MAP[source].update(target_addresses)
addresses_used_by_target.update(_SOURCE_FILE_TO_ADDRESS_MAP[source])
# Since the symbol is not first party, it needs to be mapped to a target.
else:
# The twitter.commons/apache.aurora packages are hopeless to comprehensively map. They have a number of issues:
# * They share common `top_level.txt` and 'namespace_packages.txt' values
# * Heuristics based off the package name are invalid
# * apache.aurora.thrift == gen.apache.aurora.[modules].{1.py, 2.py, 3.py}
# * apache.aurora.thermos == gen.apache.thermos.{a.py, b.py, c.py}
# * how to tell that gen.apache.thermos is not a module of gen.apache.aurora?
# * Consequently:
# * they clobber each other's namespace
# * There is no programmatic way to tell between modules
#
# The valid imports are trivial to determine but there is no deterministic way to map that to the package name.
# Without these, we could entirely rely on the virtualenv introspection but instead third_party_map lives.
if not self.opt_out_virtualenv_walk:
import_map = self.symbol_to_target_map
if prefix not in import_map:
# Slice the import ever shorter until we either match to a known import or run out of parts.
prefix = symbol
parts = len(prefix.split('.'))
while prefix not in import_map and parts > 1:
prefix, _ = prefix.rsplit('.', 1)
parts -= 1
else:
import_map = {}
# Both of these return a target spec string if there is a match and None otherwise.
dep = check_manually_defined(symbol, self.get_options().third_party_map) or import_map.get(prefix)
if not dep:
msg = (dedent("""While running python buildgen, a symbol was found without a known providing target.
Target: {}
Symbol: {}
""".format(target.address.spec, symbol)
))
# TODO(mateo): Make this exception fail-slow. Better to gather all bg failures and print at end.
if self.get_options().fatal:
raise PythonBuildgenError(msg)
else:
print('{}Ignoring for now since fatal errors are off'.format(msg))
else:
addresses_used_by_target.add(Address.parse(dep))
# Remove any imports from within the same module.
filtered_addresses_used_by_target = set([
addr for addr in addresses_used_by_target
if addr != target.address
])
self.adjust_target_build_file(target, filtered_addresses_used_by_target)
