def process_library_description(library_name: str, library: LibraryDescription,
all_libraries: Dict[str, LibraryDescription],
sdks: List[Version]):
# Check if SDK version check needs to be added.
sdk_version: Optional[dict] = None
if library.sdk_version:
for sdk in sdk_list:
if not library.applies_to_sdk_version(sdk):
sdk_version = library.sdk_version.to_json()
break
else:
sdk_version = None
# Get intersection part of the library for all SDKs (called base_library)
def all_sdk_libraries():
for sdk in sdk_list:
sdk_library = library.library_for_sdk_version(sdk)
if sdk_library:
yield sdk_library
base_library = Library.intersection(all_sdk_libraries())
# Find all other libraries described as patches
patches: Dict[str, dict] = {}
empty_library = Library()
for sdk in sdk_list:
# Check if patch applies to this SDK
sdk_library = library.library_for_sdk_version(sdk)
if not sdk_library:
continue
sdk_library.difference_update(base_library)
if sdk_library == empty_library:
continue
# Now that library is OK, process it
patches[str(sdk)] = sdk_library.to_json()
# Find all library dependencies
sdk_dependencies = libraries_dependencies_per_sdk({library_name: library},
all_libraries, sdks)
base_dependencies = set.intersection(*(x[1]
for x in sdk_dependencies.items()))
return {
"name": library_name,
"documentation": library.documentation,
"sdk_version": sdk_version,
"variant": library.variant.value.upper(),
"base": base_library.to_json(),
"patches": patches,
"dep_base": sorted(base_dependencies),
"dep_patches": {
str(x[0]): sorted(set.difference(x[1], base_dependencies))
for x in sdk_dependencies.items()
}
}
def test_dependencies(self):
sdk_14 = Version.from_string("14.0.0")
sdk_15 = Version.from_string("15.0.0")
sdk_16 = Version.from_string("16.0.0")
library_a = LibraryDescription(
variant=LibraryVariant.OBJECT,
library=Library(dependencies=Property(public={"b"})),
sdk_version=LibraryVersion(sdk_15),
patches=[
LibraryPatch(
operation=LibraryOperation.ADD,
sdk_version=LibraryVersion(sdk_15),
library=Library(dependencies=Property(public={"c"}))),
LibraryPatch(
operation=LibraryOperation.REMOVE,
sdk_version=LibraryVersion(sdk_16),
library=Library(dependencies=Property(public={"c"}))),
LibraryPatch(
operation=LibraryOperation.ADD,
sdk_version=LibraryVersion(sdk_16),
library=Library(dependencies=Property(public={"d"}))),
])
library_b = LibraryDescription(variant=LibraryVariant.OBJECT, )
library_c = LibraryDescription(variant=LibraryVariant.OBJECT)
library_d = LibraryDescription(
variant=LibraryVariant.OBJECT,
library=Library(dependencies=Property(public={"e"})))
library_e = LibraryDescription(variant=LibraryVariant.OBJECT)
library_f = LibraryDescription(variant=LibraryVariant.OBJECT)
all_libraries = {
"a": library_a,
"b": library_b,
"c": library_c,
"d": library_d,
"e": library_e,
"f": library_f
}
supported_sdks = [sdk_14, sdk_15, sdk_16]
result = libraries_dependencies_per_sdk({"a": library_a},
all_libraries, supported_sdks)
self.assertEqual(result[sdk_14], set())
self.assertEqual(result[sdk_15], {"a", "b", "c"})
self.assertEqual(result[sdk_16], {"a", "b", "d", "e"})
def from_json(json_value: dict) -> LibraryDescription:
validate_json(instance=json_value,
schema=LibraryDescription.json_schema)
library = LibraryDescription()
if "sdk_version" in json_value:
library._sdk_version = LibraryVersion.from_json(
json_value["sdk_version"])
if "documentation" in json_value:
library._documentation = json_value["documentation"]
if "variant" in json_value:
library._variant = LibraryVariant(json_value["variant"])
library._library = Library.from_json(json_value)
if "patches" in json_value:
for patch in json_value["patches"]:
library._patches.append(LibraryPatch.from_json(patch))
if "groups" in json_value:
library._groups = json_value["groups"]
return library
def __init__(self,
operation: LibraryOperation,
sdk_version: LibraryVersion,
library: Optional[Library] = None):
self._operation = operation
self._sdk_version = sdk_version
self._library: Library = library or Library()
def library_from_example(example: Example) -> Library:
return Library(
sources=example.get_prop(ExampleProperty.SOURCES),
includes=Property(public=example.get_prop(ExampleProperty.INCLUDES)),
cflags=Property(public=example.get_prop(ExampleProperty.CFLAGS)),
asmflags=Property(public=example.get_prop(ExampleProperty.ASMFLAGS)),
ldflags=Property(public=example.get_prop(ExampleProperty.LDFLAGS)))
def generate_libraries_dependencies(
examples: List[Example],
lib_descs: Dict[str, LibraryDescription]):
print("Generating dependencies...")
remaining_deps = len(lib_descs)
for lib_desc_name in lib_descs:
lib_desc = lib_descs[lib_desc_name]
# Get info about dependencies generation
sys.stdout.write("\rRemaining dependencies: " +
str(remaining_deps).ljust(4)
)
remaining_deps -= 1
# Function which checks if all sources from the library are included
# in the example sources
def includes_all_sources_from_library(example: Example):
example_sources = example.get_prop(ExampleProperty.SOURCES)
library_sources = lib_desc.library.sources
sources_intersection = set.intersection(
example_sources,
library_sources
)
return len(sources_intersection) == len(library_sources)
# Find library converted from all examples, which include all sources
# from the library.
intersection_lib = Library.intersection(
library_from_example(example) for example in examples
if includes_all_sources_from_library(example)
)
# Remove used resources from library
intersection_lib.difference_update(lib_desc.library)
# While there are any sources left we try to find library for them.
dependencies = Property()
while len(intersection_lib.sources) > 0:
# Take one random source and find library for it.
source = intersection_lib.sources.pop()
found_library = find_library_including_source(source, lib_descs)
if found_library == None:
print("WARNING: library not found for: " + source)
else:
dependencies.add_item(found_library[0], Access.PUBLIC)
intersection_lib.difference_update(found_library[1].library)
# Update library's dependencies:
lib_desc.library.set_prop(LibraryProperty.DEPENDENCIES, dependencies)
# Info about success
print("\nDependency generation finished!")
def test_library_patches(self):
lib_desc = LibraryDescription(
variant=LibraryVariant.OBJECT,
documentation="",
sdk_version=LibraryVersion(Version(15, 0, 0), Version(16, 0, 0)),
library=Library(sources={"s1", "s2"}),
patches=[
LibraryPatch(LibraryOperation.ADD,
LibraryVersion(Version(15, 1, 0)),
Library({"s3"})),
LibraryPatch(
LibraryOperation.ADD,
LibraryVersion(Version(15, 2, 0), Version(15, 3, 0)),
Library({"s4"})),
LibraryPatch(
LibraryOperation.REMOVE,
LibraryVersion(Version(15, 2, 1), Version(15, 4, 0)),
Library({"s1"}))
])
self.assertEqual(lib_desc.library_for_sdk_version(Version(14, 0, 0)),
None)
self.assertEqual(lib_desc.library_for_sdk_version(Version(16, 1, 0)),
None)
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 0, 2)).sources,
{'s1', 's2'})
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 1, 2)).sources,
{'s1', 's2', 's3'})
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 2, 0)).sources,
{'s1', 's2', 's3', 's4'})
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 2, 2)).sources,
{'s2', 's3', 's4'})
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 4, 0)).sources,
{'s2', 's3'})
self.assertSetEqual(
lib_desc.library_for_sdk_version(Version(15, 4, 1)).sources,
{'s1', 's2', 's3'})
def from_json(json_value: dict) -> LibraryPatch:
validate_json(instance=json_value,
schema=LibraryPatch.json_schema)
patch = LibraryPatch(
LibraryOperation(json_value["operation"]),
LibraryVersion.from_json(json_value["sdk_version"])
)
patch._library = Library.from_json(json_value)
return patch
def __init__(self,
variant: LibraryVariant = LibraryVariant.OBJECT,
documentation: str = "",
sdk_version: Optional[LibraryVersion] = None,
library: Optional[Library] = None,
patches: Optional[List[LibraryPatch]] = None):
self._variant = variant
self._documentation = documentation
self._sdk_version = sdk_version
self._library: Library = library or Library()
self._patches: List[LibraryPatch] = patches or []
def library_for_sdk_version(self, version: Version) -> Optional[Library]:
if not self.applies_to_sdk_version(version):
return None
result_library = Library()
result_library.union_update(self.library)
for patch in self.patches:
if not patch.sdk_version.applies_to(version):
continue
if patch.operation == LibraryOperation.ADD:
result_library.union_update(patch.library)
else:
result_library.difference_update(patch.library)
return result_library
def union_library() -> Library:
return Library.union((library_from_example(x) for x in examples))
def intersection_library_including_src(source: str) -> Library:
return Library.intersection(
library_from_example(e) for e in examples
if source in e.get_prop(ExampleProperty.SOURCES))
def remove_duplicates(library: Library):
for lib in libraries_iter():
library.difference_update(lib[1])
def generate_libraries(
examples: Iterable[Example], known_libraries: Dict[str, LibraryDescription]
) -> Dict[str, LibraryDescription]:
# Print what we are doing...
print("Generating modules...")
# This dict contains new, generated libraries
gen_libraries: Dict[str, LibraryDescription] = {}
gen_library_count: int = 0
# All defined libraries iterator
def libraries_iter():
for known_lib in known_libraries.items():
yield (known_lib[0], known_lib[1].library)
for gen_lib in gen_libraries.items():
yield (gen_lib[0], gen_lib[1].library)
# Remove duplicates from a library which exists in already known and
# generated libraries.
def remove_duplicates(library: Library):
for lib in libraries_iter():
library.difference_update(lib[1])
# This function finds intersection library from all examples which include
# source file.
def intersection_library_including_src(source: str) -> Library:
return Library.intersection(
library_from_example(e) for e in examples
if source in e.get_prop(ExampleProperty.SOURCES))
# This function creates union library from all examples
def union_library() -> Library:
return Library.union((library_from_example(x) for x in examples))
# Collect union library from all examples and remove entires already present
# in other libraries.
union_lib = union_library()
remove_duplicates(union_lib)
# Iterate until all sources are processed from the union library
while len(union_lib.sources) > 0:
# Show progress...
sys.stdout.write("\rRemaining sources: " +
str(len(union_lib.sources)).ljust(4))
# Pick first source at random
union_lib_src = next(iter(union_lib.sources))
# Get intersection library from all libraries which include union_lib_src
library = intersection_library_including_src(union_lib_src)
remove_duplicates(library)
# Check if every source in the above library always appears in every
# known example. That would suggest sources are part of the same lib.
smaller_lib_found = True
while smaller_lib_found:
# Let's assume we didn't find smaller library...
smaller_lib_found = False
# Iterate all library's sources.
for lib_src in library.sources:
# Find intersection library from all examples which includes lib_src
other_library = intersection_library_including_src(lib_src)
remove_duplicates(other_library)
# Intersect two libraries and check if sources remain the same.
merged_library = Library.intersection([library, other_library])
if len(merged_library.sources) != len(library.sources):
smaller_lib_found = True
library = merged_library
break
# Name library and add it to generated libraries
gen_libraries["lib" + str(gen_library_count)] = \
LibraryDescription(library=library)
gen_library_count += 1
# Removed used sources and other props from union_lib
union_lib.difference_update(library)
# Validate all libraries by checking that all sources, includes etc. are
# covered by gathered libs. These are only warnings.
union_lib_from_examples = union_library()
union_lib_from_libs = Library.union(x[1] for x in libraries_iter())
# Check differences
print("\nFinished processing all sources!")
union_lib_from_examples.difference_update(union_lib_from_libs)
if len(union_lib_from_examples.sources) > 0:
print("WARNING: sources not handled:")
pp.pprint(union_lib_from_examples.sources)
for prop in LibraryProperty:
all_items = union_lib_from_examples.get_prop(prop).get_all_items()
if len(all_items) > 0:
print("WARNING: " + prop.value + " not handled:")
pp.pprint(all_items)
# Return generated libs
return gen_libraries