def process_library_description(library_name: str, library: LibraryDescription, all_libraries: Dict[str, LibraryDescription], sdks: List[Version]): # Check if SDK version check needs to be added. sdk_version: Optional[dict] = None if library.sdk_version: for sdk in sdk_list: if not library.applies_to_sdk_version(sdk): sdk_version = library.sdk_version.to_json() break else: sdk_version = None # Get intersection part of the library for all SDKs (called base_library) def all_sdk_libraries(): for sdk in sdk_list: sdk_library = library.library_for_sdk_version(sdk) if sdk_library: yield sdk_library base_library = Library.intersection(all_sdk_libraries()) # Find all other libraries described as patches patches: Dict[str, dict] = {} empty_library = Library() for sdk in sdk_list: # Check if patch applies to this SDK sdk_library = library.library_for_sdk_version(sdk) if not sdk_library: continue sdk_library.difference_update(base_library) if sdk_library == empty_library: continue # Now that library is OK, process it patches[str(sdk)] = sdk_library.to_json() # Find all library dependencies sdk_dependencies = libraries_dependencies_per_sdk({library_name: library}, all_libraries, sdks) base_dependencies = set.intersection(*(x[1] for x in sdk_dependencies.items())) return { "name": library_name, "documentation": library.documentation, "sdk_version": sdk_version, "variant": library.variant.value.upper(), "base": base_library.to_json(), "patches": patches, "dep_base": sorted(base_dependencies), "dep_patches": { str(x[0]): sorted(set.difference(x[1], base_dependencies)) for x in sdk_dependencies.items() } }
def test_dependencies(self): sdk_14 = Version.from_string("14.0.0") sdk_15 = Version.from_string("15.0.0") sdk_16 = Version.from_string("16.0.0") library_a = LibraryDescription( variant=LibraryVariant.OBJECT, library=Library(dependencies=Property(public={"b"})), sdk_version=LibraryVersion(sdk_15), patches=[ LibraryPatch( operation=LibraryOperation.ADD, sdk_version=LibraryVersion(sdk_15), library=Library(dependencies=Property(public={"c"}))), LibraryPatch( operation=LibraryOperation.REMOVE, sdk_version=LibraryVersion(sdk_16), library=Library(dependencies=Property(public={"c"}))), LibraryPatch( operation=LibraryOperation.ADD, sdk_version=LibraryVersion(sdk_16), library=Library(dependencies=Property(public={"d"}))), ]) library_b = LibraryDescription(variant=LibraryVariant.OBJECT, ) library_c = LibraryDescription(variant=LibraryVariant.OBJECT) library_d = LibraryDescription( variant=LibraryVariant.OBJECT, library=Library(dependencies=Property(public={"e"}))) library_e = LibraryDescription(variant=LibraryVariant.OBJECT) library_f = LibraryDescription(variant=LibraryVariant.OBJECT) all_libraries = { "a": library_a, "b": library_b, "c": library_c, "d": library_d, "e": library_e, "f": library_f } supported_sdks = [sdk_14, sdk_15, sdk_16] result = libraries_dependencies_per_sdk({"a": library_a}, all_libraries, supported_sdks) self.assertEqual(result[sdk_14], set()) self.assertEqual(result[sdk_15], {"a", "b", "c"}) self.assertEqual(result[sdk_16], {"a", "b", "d", "e"})
def from_json(json_value: dict) -> LibraryDescription: validate_json(instance=json_value, schema=LibraryDescription.json_schema) library = LibraryDescription() if "sdk_version" in json_value: library._sdk_version = LibraryVersion.from_json( json_value["sdk_version"]) if "documentation" in json_value: library._documentation = json_value["documentation"] if "variant" in json_value: library._variant = LibraryVariant(json_value["variant"]) library._library = Library.from_json(json_value) if "patches" in json_value: for patch in json_value["patches"]: library._patches.append(LibraryPatch.from_json(patch)) if "groups" in json_value: library._groups = json_value["groups"] return library
def __init__(self, operation: LibraryOperation, sdk_version: LibraryVersion, library: Optional[Library] = None): self._operation = operation self._sdk_version = sdk_version self._library: Library = library or Library()
def library_from_example(example: Example) -> Library: return Library( sources=example.get_prop(ExampleProperty.SOURCES), includes=Property(public=example.get_prop(ExampleProperty.INCLUDES)), cflags=Property(public=example.get_prop(ExampleProperty.CFLAGS)), asmflags=Property(public=example.get_prop(ExampleProperty.ASMFLAGS)), ldflags=Property(public=example.get_prop(ExampleProperty.LDFLAGS)))
def generate_libraries_dependencies( examples: List[Example], lib_descs: Dict[str, LibraryDescription]): print("Generating dependencies...") remaining_deps = len(lib_descs) for lib_desc_name in lib_descs: lib_desc = lib_descs[lib_desc_name] # Get info about dependencies generation sys.stdout.write("\rRemaining dependencies: " + str(remaining_deps).ljust(4) ) remaining_deps -= 1 # Function which checks if all sources from the library are included # in the example sources def includes_all_sources_from_library(example: Example): example_sources = example.get_prop(ExampleProperty.SOURCES) library_sources = lib_desc.library.sources sources_intersection = set.intersection( example_sources, library_sources ) return len(sources_intersection) == len(library_sources) # Find library converted from all examples, which include all sources # from the library. intersection_lib = Library.intersection( library_from_example(example) for example in examples if includes_all_sources_from_library(example) ) # Remove used resources from library intersection_lib.difference_update(lib_desc.library) # While there are any sources left we try to find library for them. dependencies = Property() while len(intersection_lib.sources) > 0: # Take one random source and find library for it. source = intersection_lib.sources.pop() found_library = find_library_including_source(source, lib_descs) if found_library == None: print("WARNING: library not found for: " + source) else: dependencies.add_item(found_library[0], Access.PUBLIC) intersection_lib.difference_update(found_library[1].library) # Update library's dependencies: lib_desc.library.set_prop(LibraryProperty.DEPENDENCIES, dependencies) # Info about success print("\nDependency generation finished!")
def test_library_patches(self): lib_desc = LibraryDescription( variant=LibraryVariant.OBJECT, documentation="", sdk_version=LibraryVersion(Version(15, 0, 0), Version(16, 0, 0)), library=Library(sources={"s1", "s2"}), patches=[ LibraryPatch(LibraryOperation.ADD, LibraryVersion(Version(15, 1, 0)), Library({"s3"})), LibraryPatch( LibraryOperation.ADD, LibraryVersion(Version(15, 2, 0), Version(15, 3, 0)), Library({"s4"})), LibraryPatch( LibraryOperation.REMOVE, LibraryVersion(Version(15, 2, 1), Version(15, 4, 0)), Library({"s1"})) ]) self.assertEqual(lib_desc.library_for_sdk_version(Version(14, 0, 0)), None) self.assertEqual(lib_desc.library_for_sdk_version(Version(16, 1, 0)), None) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 0, 2)).sources, {'s1', 's2'}) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 1, 2)).sources, {'s1', 's2', 's3'}) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 2, 0)).sources, {'s1', 's2', 's3', 's4'}) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 2, 2)).sources, {'s2', 's3', 's4'}) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 4, 0)).sources, {'s2', 's3'}) self.assertSetEqual( lib_desc.library_for_sdk_version(Version(15, 4, 1)).sources, {'s1', 's2', 's3'})
def from_json(json_value: dict) -> LibraryPatch: validate_json(instance=json_value, schema=LibraryPatch.json_schema) patch = LibraryPatch( LibraryOperation(json_value["operation"]), LibraryVersion.from_json(json_value["sdk_version"]) ) patch._library = Library.from_json(json_value) return patch
def __init__(self, variant: LibraryVariant = LibraryVariant.OBJECT, documentation: str = "", sdk_version: Optional[LibraryVersion] = None, library: Optional[Library] = None, patches: Optional[List[LibraryPatch]] = None): self._variant = variant self._documentation = documentation self._sdk_version = sdk_version self._library: Library = library or Library() self._patches: List[LibraryPatch] = patches or []
def library_for_sdk_version(self, version: Version) -> Optional[Library]: if not self.applies_to_sdk_version(version): return None result_library = Library() result_library.union_update(self.library) for patch in self.patches: if not patch.sdk_version.applies_to(version): continue if patch.operation == LibraryOperation.ADD: result_library.union_update(patch.library) else: result_library.difference_update(patch.library) return result_library
def union_library() -> Library: return Library.union((library_from_example(x) for x in examples))
def intersection_library_including_src(source: str) -> Library: return Library.intersection( library_from_example(e) for e in examples if source in e.get_prop(ExampleProperty.SOURCES))
def remove_duplicates(library: Library): for lib in libraries_iter(): library.difference_update(lib[1])
def generate_libraries( examples: Iterable[Example], known_libraries: Dict[str, LibraryDescription] ) -> Dict[str, LibraryDescription]: # Print what we are doing... print("Generating modules...") # This dict contains new, generated libraries gen_libraries: Dict[str, LibraryDescription] = {} gen_library_count: int = 0 # All defined libraries iterator def libraries_iter(): for known_lib in known_libraries.items(): yield (known_lib[0], known_lib[1].library) for gen_lib in gen_libraries.items(): yield (gen_lib[0], gen_lib[1].library) # Remove duplicates from a library which exists in already known and # generated libraries. def remove_duplicates(library: Library): for lib in libraries_iter(): library.difference_update(lib[1]) # This function finds intersection library from all examples which include # source file. def intersection_library_including_src(source: str) -> Library: return Library.intersection( library_from_example(e) for e in examples if source in e.get_prop(ExampleProperty.SOURCES)) # This function creates union library from all examples def union_library() -> Library: return Library.union((library_from_example(x) for x in examples)) # Collect union library from all examples and remove entires already present # in other libraries. union_lib = union_library() remove_duplicates(union_lib) # Iterate until all sources are processed from the union library while len(union_lib.sources) > 0: # Show progress... sys.stdout.write("\rRemaining sources: " + str(len(union_lib.sources)).ljust(4)) # Pick first source at random union_lib_src = next(iter(union_lib.sources)) # Get intersection library from all libraries which include union_lib_src library = intersection_library_including_src(union_lib_src) remove_duplicates(library) # Check if every source in the above library always appears in every # known example. That would suggest sources are part of the same lib. smaller_lib_found = True while smaller_lib_found: # Let's assume we didn't find smaller library... smaller_lib_found = False # Iterate all library's sources. for lib_src in library.sources: # Find intersection library from all examples which includes lib_src other_library = intersection_library_including_src(lib_src) remove_duplicates(other_library) # Intersect two libraries and check if sources remain the same. merged_library = Library.intersection([library, other_library]) if len(merged_library.sources) != len(library.sources): smaller_lib_found = True library = merged_library break # Name library and add it to generated libraries gen_libraries["lib" + str(gen_library_count)] = \ LibraryDescription(library=library) gen_library_count += 1 # Removed used sources and other props from union_lib union_lib.difference_update(library) # Validate all libraries by checking that all sources, includes etc. are # covered by gathered libs. These are only warnings. union_lib_from_examples = union_library() union_lib_from_libs = Library.union(x[1] for x in libraries_iter()) # Check differences print("\nFinished processing all sources!") union_lib_from_examples.difference_update(union_lib_from_libs) if len(union_lib_from_examples.sources) > 0: print("WARNING: sources not handled:") pp.pprint(union_lib_from_examples.sources) for prop in LibraryProperty: all_items = union_lib_from_examples.get_prop(prop).get_all_items() if len(all_items) > 0: print("WARNING: " + prop.value + " not handled:") pp.pprint(all_items) # Return generated libs return gen_libraries