def _scala_library_used_addresses(self, target): """Consults the analysis products to construct a set of addresses a scala library uses.""" syms = self._symbols_used_by_scala_target(target) used_addresses = set() errors = [] for symbol in syms: exact_matching_sources = self._internal_symbol_tree.get( symbol, exact=False) manually_defined_target = check_manually_defined( symbol, subtree=self.merged_map) if manually_defined_target and exact_matching_sources: print( 'ERROR: buildgen found both sources and manually defined in third_party_map_jvm' ' targets for this symbol.\n' 'Target: {0}\n' 'Jvm symbol used by target: {1}\n' 'Manually defined target for symbol: {3}\n' 'Sources found defining symbol: \n{2}\n'.format( target.address.reference(), symbol, '\n'.join(' * {0}'.format(source) for source in exact_matching_sources), self._manually_defined_spec_to_address( manually_defined_target).reference(), )) errors.append((target.address.reference(), symbol)) continue elif exact_matching_sources: addresses = set( chain.from_iterable( self._source_mapper.target_addresses_for_source(source) for source in exact_matching_sources)) elif manually_defined_target == 'SKIP': continue elif manually_defined_target: addresses = [ self._manually_defined_spec_to_address( manually_defined_target) ] else: errors.append((target.address.reference(), symbol)) continue for address in addresses: dep = self.context.build_graph.get_target(address) if not dep: raise UsedSymbolException( "An address was used that was not injected into the build graph! Make sure that " "there is a matching BUILD definition for this used address: {}" .format(address), ) if address == target.address: pass elif self._is_test(dep) and not self._is_test(target): pass else: # In the end, we always actually depend on concrete targets. But for now we preserve # the information that this dependency (could have been) synthetic, and let downstream # consumers normalize this to a concrete target if necessary. used_addresses.add(dep.address) if errors: print(self._internal_symbol_tree.render()) print( 'ERROR: Failed to map these symbols used by the following target to a providing' ' target:', file=sys.stderr) err_msg = [] for spec, symbol in errors: err_msg.append("") err_msg.append("Symbol: " + symbol) err_msg.append("Target: " + spec) err_msg.append( 'Failed to map scala libraries to used symbols. See error output above.' ) raise Exception('\n'.join(err_msg)) return used_addresses
def _scala_library_used_addresses(self, target): """Consults the analysis products to construct a set of addresses a scala library uses.""" syms = self._symbols_used_by_scala_target(target) used_addresses = set() errors = [] for symbol in syms: exact_matching_sources = self._internal_symbol_tree.get(symbol, exact=False) manually_defined_target = check_manually_defined(symbol, subtree=self.merged_map) if manually_defined_target and exact_matching_sources: print( 'ERROR: buildgen found both sources and manually defined in third_party_map_jvm' ' targets for this symbol.\n' 'Target: {0}\n' 'Jvm symbol used by target: {1}\n' 'Manually defined target for symbol: {3}\n' 'Sources found defining symbol: \n{2}\n' .format( target.address.reference(), symbol, '\n'.join(' * {0}'.format(source) for source in exact_matching_sources), self._manually_defined_spec_to_address(manually_defined_target).reference(), ) ) errors.append((target.address.reference(), symbol)) continue elif exact_matching_sources: addresses = set(chain.from_iterable( self._source_mapper.target_addresses_for_source(source) for source in exact_matching_sources )) elif manually_defined_target == 'SKIP': continue elif manually_defined_target: addresses = [self._manually_defined_spec_to_address(manually_defined_target)] else: errors.append((target.address.reference(), symbol)) continue for address in addresses: dep = self.context.build_graph.get_target(address) # NOTE(mateo): This cannot happen when using pom-resolve, but OSS consumers have been bitten when dep is None. # I was unable to repro using Ivy, but this check is cheap enough to be worth it no matter the resolver. if not dep: raise UsedSymbolException("An address was used that was not injected into the build graph! Make sure that " "there is a matching BUILD definition for this used address: {}".format(address)) if address == target.address: pass elif self._is_test(dep) and not self._is_test(target): pass else: # In the end, we always actually depend on concrete targets. But for now we preserve # the information that this dependency (could have been) synthetic, and let downstream # consumers normalize this to a concrete target if necessary. used_addresses.add(dep.address) if errors: print('ERROR: Failed to map these symbols used by the following target to a providing' ' target:', file=sys.stderr) err_msg = [] for spec, symbol in errors: err_msg.append("") err_msg.append("Symbol: " + symbol) err_msg.append("Target: " + spec) err_msg.append('Failed to map scala libraries to used symbols. See error output above.') raise Exception('\n'.join(err_msg)) return used_addresses
def buildgen_target(self, target): safe_mkdir(self.workdir) source_files = [f for f in target.sources_relative_to_buildroot() if f.endswith('.py')] build_graph = self.context.build_graph source_to_addresses_mapper = self.context.products.get_data('source_to_addresses_mapper') # Gather symbols imported from first party source files. target_used_symbols = set() addresses_used_by_target = set() for source_candidate in source_files: try: used_symbols = self.get_used_symbols(source_candidate) except Exception as e: raise Exception( dedent( """ {}\n Source file: {} Referenced by: {} """.format(e, source_candidate, target.address.spec) ) ) target_used_symbols.update(used_symbols) for symbol in target_used_symbols: prefix = symbol.split('.')[0] # If the symbol is a first party package, map it to providing source files and memoize the relation. if prefix in self.first_party_packages: if symbol not in _SYMBOLS_TO_SOURCES_MAP: providing_sources = self.symbol_to_source_tree.get(symbol, allow_prefix_imports=True) if not providing_sources: raise Exception( 'While python buildgenning {}, encountered a symbol with' ' no providing target. This probably means the import moved' ' or is misspelled. It could also mean that there is no BUILD' ' target that owns the source that provides the symbol.' ' Imported symbol: {}' .format(target.address.spec, symbol) ) _SYMBOLS_TO_SOURCES_MAP[symbol] = providing_sources # Map source file to concrete addresses, tracing codegen back to its concrete target, and cache it. for source in _SYMBOLS_TO_SOURCES_MAP[symbol]: if source not in _SOURCE_FILE_TO_ADDRESS_MAP: target_addresses = set() for address in source_to_addresses_mapper.target_addresses_for_source(source): concrete = build_graph.get_concrete_derived_from(address) if concrete.type_alias != 'python_binary': target_addresses.add(concrete.address) _SOURCE_FILE_TO_ADDRESS_MAP[source].update(target_addresses) addresses_used_by_target.update(_SOURCE_FILE_TO_ADDRESS_MAP[source]) # Since the symbol is not first party, it needs to be mapped to a target. else: # The twitter.commons/apache.aurora packages are hopeless to comprehensively map. They have a number of issues: # * They share common `top_level.txt` and 'namespace_packages.txt' values # * Heuristics based off the package name are invalid # * apache.aurora.thrift == gen.apache.aurora.[modules].{1.py, 2.py, 3.py} # * apache.aurora.thermos == gen.apache.thermos.{a.py, b.py, c.py} # * how to tell that gen.apache.thermos is not a module of gen.apache.aurora? # * Consequently: # * they clobber each other's namespace # * There is no programmatic way to tell between modules # # The valid imports are trivial to determine but there is no deterministic way to map that to the package name. # Without these, we could entirely rely on the virtualenv introspection but instead third_party_map lives. if not self.opt_out_virtualenv_walk: import_map = self.symbol_to_target_map if prefix not in import_map: # Slice the import ever shorter until we either match to a known import or run out of parts. prefix = symbol parts = len(prefix.split('.')) while prefix not in import_map and parts > 1: prefix, _ = prefix.rsplit('.', 1) parts -= 1 else: import_map = {} # Both of these return a target spec string if there is a match and None otherwise. dep = check_manually_defined(symbol, self.get_options().third_party_map) or import_map.get(prefix) if not dep: msg = dedent( """\ While running python buildgen, a symbol was found without a known providing target. Target: {} Symbol: {} """.format(target.address.spec, symbol) ) # TODO(mateo): Make this exception fail-slow. Better to gather all bg failures and print at end. if self.get_options().fatal: raise PythonBuildgenError(msg) else: print('{}Ignoring for now since fatal errors are off'.format(msg)) else: addresses_used_by_target.add(Address.parse(dep)) # Remove any imports from within the same module. filtered_addresses_used_by_target = { addr for addr in addresses_used_by_target if addr != target.address } self.adjust_target_build_file(target, filtered_addresses_used_by_target)
def buildgen_target(self, target): safe_mkdir(self.workdir) source_files = [f for f in target.sources_relative_to_buildroot() if f.endswith('.py')] build_graph = self.context.build_graph address_mapper = self.context.address_mapper source_to_addresses_mapper = self.context.products.get_data('source_to_addresses_mapper') # Gather symbols imported from first party source files. target_used_symbols = set() addresses_used_by_target = set() for source_candidate in source_files: target_used_symbols.update(self.get_used_symbols(source_candidate)) for symbol in target_used_symbols: prefix = symbol.split('.')[0] # If the symbol is a first party package, map it to providing source files and memoize the relation. if prefix in self.first_party_packages: if symbol not in _SYMBOLS_TO_SOURCES_MAP: providing_sources = self.symbol_to_source_tree.get(symbol, allow_prefix_imports=True) if not providing_sources: raise Exception( 'While python buildgenning {}, encountered a symbol with' ' no providing target. This probably means the import moved' ' or is misspelled. It could also mean that there is no BUILD' ' target that owns the source that provides the symbol.' ' Imported symbol: {}' .format(target.address.spec, symbol) ) _SYMBOLS_TO_SOURCES_MAP[symbol] = providing_sources # Map source file to concrete addresses, tracing codegen back to its concrete target, and cache it. for source in _SYMBOLS_TO_SOURCES_MAP[symbol]: if source not in _SOURCE_FILE_TO_ADDRESS_MAP: target_addresses = set() for address in source_to_addresses_mapper.target_addresses_for_source(source): concrete = build_graph.get_concrete_derived_from(address) if concrete.type_alias != 'python_binary': target_addresses.add(concrete.address) _SOURCE_FILE_TO_ADDRESS_MAP[source].update(target_addresses) addresses_used_by_target.update(_SOURCE_FILE_TO_ADDRESS_MAP[source]) # Since the symbol is not first party, it needs to be mapped to a target. else: # The twitter.commons/apache.aurora packages are hopeless to comprehensively map. They have a number of issues: # * They share common `top_level.txt` and 'namespace_packages.txt' values # * Heuristics based off the package name are invalid # * apache.aurora.thrift == gen.apache.aurora.[modules].{1.py, 2.py, 3.py} # * apache.aurora.thermos == gen.apache.thermos.{a.py, b.py, c.py} # * how to tell that gen.apache.thermos is not a module of gen.apache.aurora? # * Consequently: # * they clobber each other's namespace # * There is no programmatic way to tell between modules # # The valid imports are trivial to determine but there is no deterministic way to map that to the package name. # Without these, we could entirely rely on the virtualenv introspection but instead third_party_map lives. if not self.opt_out_virtualenv_walk: import_map = self.symbol_to_target_map if prefix not in import_map: # Slice the import ever shorter until we either match to a known import or run out of parts. prefix = symbol parts = len(prefix.split('.')) while prefix not in import_map and parts > 1: prefix, _ = prefix.rsplit('.', 1) parts -= 1 else: import_map = {} # Both of these return a target spec string if there is a match and None otherwise. dep = check_manually_defined(symbol, self.get_options().third_party_map) or import_map.get(prefix) if not dep: msg = (dedent("""While running python buildgen, a symbol was found without a known providing target. Target: {} Symbol: {} """.format(target.address.spec, symbol) )) # TODO(mateo): Make this exception fail-slow. Better to gather all bg failures and print at end. if self.get_options().fatal: raise PythonBuildgenError(msg) else: print('{}Ignoring for now since fatal errors are off'.format(msg)) else: addresses_used_by_target.add(Address.parse(dep)) # Remove any imports from within the same module. filtered_addresses_used_by_target = set([ addr for addr in addresses_used_by_target if addr != target.address ]) self.adjust_target_build_file(target, filtered_addresses_used_by_target)