def process_library_description(library_name: str, library: LibraryDescription,
all_libraries: Dict[str, LibraryDescription],
sdks: List[Version]):
# Check if SDK version check needs to be added.
sdk_version: Optional[dict] = None
if library.sdk_version:
for sdk in sdk_list:
if not library.applies_to_sdk_version(sdk):
sdk_version = library.sdk_version.to_json()
break
else:
sdk_version = None
# Get intersection part of the library for all SDKs (called base_library)
def all_sdk_libraries():
for sdk in sdk_list:
sdk_library = library.library_for_sdk_version(sdk)
if sdk_library:
yield sdk_library
base_library = Library.intersection(all_sdk_libraries())
# Find all other libraries described as patches
patches: Dict[str, dict] = {}
empty_library = Library()
for sdk in sdk_list:
# Check if patch applies to this SDK
sdk_library = library.library_for_sdk_version(sdk)
if not sdk_library:
continue
sdk_library.difference_update(base_library)
if sdk_library == empty_library:
continue
# Now that library is OK, process it
patches[str(sdk)] = sdk_library.to_json()
# Find all library dependencies
sdk_dependencies = libraries_dependencies_per_sdk({library_name: library},
all_libraries, sdks)
base_dependencies = set.intersection(*(x[1]
for x in sdk_dependencies.items()))
return {
"name": library_name,
"documentation": library.documentation,
"sdk_version": sdk_version,
"variant": library.variant.value.upper(),
"base": base_library.to_json(),
"patches": patches,
"dep_base": sorted(base_dependencies),
"dep_patches": {
str(x[0]): sorted(set.difference(x[1], base_dependencies))
for x in sdk_dependencies.items()
}
}
def is_library_description_ok(library: LibraryDescription, sdks: List[Version]) -> bool:
# We ignore builtin libraries.
if library.variant == LibraryVariant.BUILTIN:
return False
# Library needs to apply to at least one SDK version.
for sdk in sdks:
if library.applies_to_sdk_version(sdk):
return True
return False
def libraries_load_from_file(filepath: str) -> Dict[str, LibraryDescription]:
libs: Dict[str, LibraryDescription] = {}
with open(filepath, 'r') as file:
json_libs: Dict[str, Any] = json.load(file)
if not isinstance(json_libs, dict):
raise Exception("Exampected a dictionary of libraries")
for json_lib_name in json_libs:
json_lib = json_libs[json_lib_name]
libs[json_lib_name] = LibraryDescription.from_json(json_lib)
return libs
def test_dependencies(self):
sdk_14 = Version.from_string("14.0.0")
sdk_15 = Version.from_string("15.0.0")
sdk_16 = Version.from_string("16.0.0")
library_a = LibraryDescription(
variant=LibraryVariant.OBJECT,
library=Library(dependencies=Property(public={"b"})),
sdk_version=LibraryVersion(sdk_15),
patches=[
LibraryPatch(
operation=LibraryOperation.ADD,
sdk_version=LibraryVersion(sdk_15),
library=Library(dependencies=Property(public={"c"}))),
LibraryPatch(
operation=LibraryOperation.REMOVE,
sdk_version=LibraryVersion(sdk_16),
library=Library(dependencies=Property(public={"c"}))),
LibraryPatch(
operation=LibraryOperation.ADD,
sdk_version=LibraryVersion(sdk_16),
library=Library(dependencies=Property(public={"d"}))),
])
library_b = LibraryDescription(variant=LibraryVariant.OBJECT, )
library_c = LibraryDescription(variant=LibraryVariant.OBJECT)
library_d = LibraryDescription(
variant=LibraryVariant.OBJECT,
library=Library(dependencies=Property(public={"e"})))
library_e = LibraryDescription(variant=LibraryVariant.OBJECT)
library_f = LibraryDescription(variant=LibraryVariant.OBJECT)
all_libraries = {
"a": library_a,
"b": library_b,
"c": library_c,
"d": library_d,
"e": library_e,
"f": library_f
}
supported_sdks = [sdk_14, sdk_15, sdk_16]
result = libraries_dependencies_per_sdk({"a": library_a},
all_libraries, supported_sdks)
self.assertEqual(result[sdk_14], set())
self.assertEqual(result[sdk_15], {"a", "b", "c"})
self.assertEqual(result[sdk_16], {"a", "b", "d", "e"})
def generate_libraries(
examples: Iterable[Example], known_libraries: Dict[str, LibraryDescription]
) -> Dict[str, LibraryDescription]:
# Print what we are doing...
print("Generating modules...")
# This dict contains new, generated libraries
gen_libraries: Dict[str, LibraryDescription] = {}
gen_library_count: int = 0
# All defined libraries iterator
def libraries_iter():
for known_lib in known_libraries.items():
yield (known_lib[0], known_lib[1].library)
for gen_lib in gen_libraries.items():
yield (gen_lib[0], gen_lib[1].library)
# Remove duplicates from a library which exists in already known and
# generated libraries.
def remove_duplicates(library: Library):
for lib in libraries_iter():
library.difference_update(lib[1])
# This function finds intersection library from all examples which include
# source file.
def intersection_library_including_src(source: str) -> Library:
return Library.intersection(
library_from_example(e) for e in examples
if source in e.get_prop(ExampleProperty.SOURCES))
# This function creates union library from all examples
def union_library() -> Library:
return Library.union((library_from_example(x) for x in examples))
# Collect union library from all examples and remove entires already present
# in other libraries.
union_lib = union_library()
remove_duplicates(union_lib)
# Iterate until all sources are processed from the union library
while len(union_lib.sources) > 0:
# Show progress...
sys.stdout.write("\rRemaining sources: " +
str(len(union_lib.sources)).ljust(4))
# Pick first source at random
union_lib_src = next(iter(union_lib.sources))
# Get intersection library from all libraries which include union_lib_src
library = intersection_library_including_src(union_lib_src)
remove_duplicates(library)
# Check if every source in the above library always appears in every
# known example. That would suggest sources are part of the same lib.
smaller_lib_found = True
while smaller_lib_found:
# Let's assume we didn't find smaller library...
smaller_lib_found = False
# Iterate all library's sources.
for lib_src in library.sources:
# Find intersection library from all examples which includes lib_src
other_library = intersection_library_including_src(lib_src)
remove_duplicates(other_library)
# Intersect two libraries and check if sources remain the same.
merged_library = Library.intersection([library, other_library])
if len(merged_library.sources) != len(library.sources):
smaller_lib_found = True
library = merged_library
break
# Name library and add it to generated libraries
gen_libraries["lib" + str(gen_library_count)] = \
LibraryDescription(library=library)
gen_library_count += 1
# Removed used sources and other props from union_lib
union_lib.difference_update(library)
# Validate all libraries by checking that all sources, includes etc. are
# covered by gathered libs. These are only warnings.
union_lib_from_examples = union_library()
union_lib_from_libs = Library.union(x[1] for x in libraries_iter())
# Check differences
print("\nFinished processing all sources!")
union_lib_from_examples.difference_update(union_lib_from_libs)
if len(union_lib_from_examples.sources) > 0:
print("WARNING: sources not handled:")
pp.pprint(union_lib_from_examples.sources)
for prop in LibraryProperty:
all_items = union_lib_from_examples.get_prop(prop).get_all_items()
if len(all_items) > 0:
print("WARNING: " + prop.value + " not handled:")
pp.pprint(all_items)
# Return generated libs
return gen_libraries
def process_library(file_path: str,
library_descriptions: Dict[str, LibraryDescription]):
with open(file_path, 'r') as file:
content = file.read()
# Create a dict of libraries.
new_lib_desc_descriptions: Dict[str, LibraryDescription] = {}
# Search for all libraries and add them.
library_names = re.findall(
r"(?:\#(.*))?\s*add_library\((\S+)\s*(\S+)\s*(?:EXCLUDE_FROM_ALL)?((?:\s*\"\S+\")*)\s*\)", content)
for (documentation, library_name, library_type, library_sources) in library_names:
# Check library type
if not library_type in ["OBJECT", "INTERFACE"]:
print("Library not handled: " + library_name +
" with type: " + library_type)
continue
# Create a new library
new_lib_desc = LibraryDescription(
variant=LibraryVariant(library_type.lower())
)
# Add sources if are relative to nRF SDK path.
library_sources = library_sources.split()
for library_source in library_sources:
library_source = library_source.strip("\"")
prefix = "${NRF5_SDK_PATH}/"
if library_source.startswith(prefix):
new_lib_desc.library.sources.add(
library_source[len(prefix):]
)
# Add library
if documentation != None:
new_lib_desc.documentation = documentation.strip()
new_lib_desc_descriptions[library_name] = new_lib_desc
# Search for new include flags
library_includes = re.findall(
r"target_include_directories\((\S+)\s*(\S+)((?:\s+\"\S+\")+)\s*\)", content)
for (library_name, access_modifier, includes) in library_includes:
# Ignore invalid access modifiers
if not access_modifier in ["PUBLIC", "INTERFACE", "PRIVATE"]:
print("WARNING: Unknown access modifier: " + access_modifier)
continue
# Ignore not registered libraries
if not library_name in new_lib_desc_descriptions:
print("WARNING: Library not found: " + library_name)
continue
# Add include paths to known library
includes = includes.split()
for include in includes:
include = include.strip("\"").strip("/")
prefix = "${NRF5_SDK_PATH}/"
if include.startswith(prefix):
lib_desc = new_lib_desc_descriptions[library_name]
lib_desc.library.get_prop(
LibraryProperty.INCLUDES
).add_item(
include[len(prefix):],
Access(access_modifier.lower())
)
# Search for dependencies
library_dependencies = re.findall(
r"target_link_libraries\((\w+)\s*(\w+)\s*((?:\w\s*)+)\)", content)
for (library_name, access_modifier, dependencies) in library_dependencies:
# Ignore invalid access modifiers
if not access_modifier in ["PUBLIC", "INTERFACE", "PRIVATE"]:
print("WARNING: Unknown access modifier: " +
access_modifier + " for " + library_name)
continue
# Ignore not registered libraries
if not library_name in new_lib_desc_descriptions:
print("WARNING: Library not found: " + library_name)
continue
# Add dependencies to known library
dependencies = dependencies.split()
for dependency in dependencies:
dependency = dependency.strip()
lib_desc = new_lib_desc_descriptions[library_name]
lib_desc.library.get_prop(
LibraryProperty.DEPENDENCIES
).add_item(
dependency,
Access(access_modifier.lower())
)
# Add new libraries
for name, lib in new_lib_desc_descriptions.items():
library_descriptions[name] = lib
def generate_library_test(library_name: str, library: LibraryDescription,
libraries: Dict[str, LibraryDescription],
supported_sdks: List[str]):
# Collect all dependencies
custom_patch: bool = False
# Collect a list of dependencies for each SDK version.
sdk_dependencies: Dict[str, Set[str]] = {}
for sdk_version in supported_sdks:
# Get list of all dependencies
version = Version.from_string(sdk_version)
library_for_sdk = library.library_for_sdk_version(version)
if library_for_sdk == None:
sdk_dependencies[sdk_version] = set()
continue
# Get all dependencies (merge interface & private)
dependencies = library_for_sdk.get_prop(
LibraryProperty.DEPENDENCIES).get_all_items()
dependencies.add(library_name)
# Custom patches to make tests buildable with optional deps.
custom_patches = {
"nrf5_ble_lesc": {"nrf5_crypto_cc310_backend"},
"nrf5_fds": {"nrf5_fstorage_sd"},
"nrf5_ble_peer_data_storage": {"nrf5_ble_peer_manager"},
"nrf5_ble_peer_manager": {"nrf5_fstorage_sd"},
"nrf5_log_default_backends":
{"nrf5_log_backend_uart", "nrf5_log_backend_serial"}
}
if library_name in custom_patches:
custom_patch = True
dependencies.update(custom_patches[library_name])
# Iterate over all existing dependencies and collect new ones.
# If expanded list of dependencies is bigger than original ones
# continue.
while True:
new_dependencies = dependencies.copy()
for dependency in dependencies:
# Check if dependecy exists...
if not dependency in all_libraries:
print(f"WARNING: dependency {dependency} doesn't exist")
continue
library_dep_desc = all_libraries[dependency]
# Check if dependency exists for this SDK version.
library_dep = library_dep_desc.library_for_sdk_version(version)
if library_dep == None:
print(
f"WARNING: dependency {dependency} should exist for SDK {version} inside {library_name}"
)
continue
# Get all dependencies and apply them.
library_dep_dep_list = library_dep.get_prop(
LibraryProperty.DEPENDENCIES).get_all_items()
new_dependencies.update(library_dep_dep_list)
# Check if two sets are the same
if new_dependencies == dependencies:
break
# Use new extended list of dependencies.
dependencies = new_dependencies
# Add generated dependencies to version
sdk_dependencies[sdk_version] = dependencies
# Generate base dependencies.
base_dependencies = set.intersection(*(x[1]
for x in sdk_dependencies.items()))
# Add SDK version
sdk_version = None
if library.sdk_version:
sdk_version = library.sdk_version.to_json()
# Return data used to create a library test.
return {
"name": library_name,
"custom_patch": custom_patch,
"sdk_version": sdk_version,
"base": sorted(base_dependencies),
"patches": {
x[0]: sorted(set.difference(x[1], base_dependencies))
for x in sdk_dependencies.items()
}
}