def process_library_description(library_name: str, library: LibraryDescription,
                                all_libraries: Dict[str, LibraryDescription],
                                sdks: List[Version]):

    # Check if SDK version check needs to be added.
    sdk_version: Optional[dict] = None
    if library.sdk_version:
        for sdk in sdk_list:
            if not library.applies_to_sdk_version(sdk):
                sdk_version = library.sdk_version.to_json()
                break
        else:
            sdk_version = None

    # Get intersection part of the library for all SDKs (called base_library)
    def all_sdk_libraries():
        for sdk in sdk_list:
            sdk_library = library.library_for_sdk_version(sdk)
            if sdk_library:
                yield sdk_library

    base_library = Library.intersection(all_sdk_libraries())

    # Find all other libraries described as patches
    patches: Dict[str, dict] = {}
    empty_library = Library()
    for sdk in sdk_list:
        # Check if patch applies to this SDK
        sdk_library = library.library_for_sdk_version(sdk)
        if not sdk_library:
            continue

        sdk_library.difference_update(base_library)
        if sdk_library == empty_library:
            continue

        # Now that library is OK, process it
        patches[str(sdk)] = sdk_library.to_json()

    # Find all library dependencies
    sdk_dependencies = libraries_dependencies_per_sdk({library_name: library},
                                                      all_libraries, sdks)
    base_dependencies = set.intersection(*(x[1]
                                           for x in sdk_dependencies.items()))

    return {
        "name": library_name,
        "documentation": library.documentation,
        "sdk_version": sdk_version,
        "variant": library.variant.value.upper(),
        "base": base_library.to_json(),
        "patches": patches,
        "dep_base": sorted(base_dependencies),
        "dep_patches": {
            str(x[0]): sorted(set.difference(x[1], base_dependencies))
            for x in sdk_dependencies.items()
        }
    }
Example #2
0
    def test_dependencies(self):

        sdk_14 = Version.from_string("14.0.0")
        sdk_15 = Version.from_string("15.0.0")
        sdk_16 = Version.from_string("16.0.0")

        library_a = LibraryDescription(
            variant=LibraryVariant.OBJECT,
            library=Library(dependencies=Property(public={"b"})),
            sdk_version=LibraryVersion(sdk_15),
            patches=[
                LibraryPatch(
                    operation=LibraryOperation.ADD,
                    sdk_version=LibraryVersion(sdk_15),
                    library=Library(dependencies=Property(public={"c"}))),
                LibraryPatch(
                    operation=LibraryOperation.REMOVE,
                    sdk_version=LibraryVersion(sdk_16),
                    library=Library(dependencies=Property(public={"c"}))),
                LibraryPatch(
                    operation=LibraryOperation.ADD,
                    sdk_version=LibraryVersion(sdk_16),
                    library=Library(dependencies=Property(public={"d"}))),
            ])

        library_b = LibraryDescription(variant=LibraryVariant.OBJECT, )

        library_c = LibraryDescription(variant=LibraryVariant.OBJECT)

        library_d = LibraryDescription(
            variant=LibraryVariant.OBJECT,
            library=Library(dependencies=Property(public={"e"})))

        library_e = LibraryDescription(variant=LibraryVariant.OBJECT)

        library_f = LibraryDescription(variant=LibraryVariant.OBJECT)

        all_libraries = {
            "a": library_a,
            "b": library_b,
            "c": library_c,
            "d": library_d,
            "e": library_e,
            "f": library_f
        }

        supported_sdks = [sdk_14, sdk_15, sdk_16]

        result = libraries_dependencies_per_sdk({"a": library_a},
                                                all_libraries, supported_sdks)

        self.assertEqual(result[sdk_14], set())
        self.assertEqual(result[sdk_15], {"a", "b", "c"})
        self.assertEqual(result[sdk_16], {"a", "b", "d", "e"})
    def from_json(json_value: dict) -> LibraryDescription:
        validate_json(instance=json_value,
                      schema=LibraryDescription.json_schema)

        library = LibraryDescription()

        if "sdk_version" in json_value:
            library._sdk_version = LibraryVersion.from_json(
                json_value["sdk_version"])

        if "documentation" in json_value:
            library._documentation = json_value["documentation"]

        if "variant" in json_value:
            library._variant = LibraryVariant(json_value["variant"])

        library._library = Library.from_json(json_value)

        if "patches" in json_value:
            for patch in json_value["patches"]:
                library._patches.append(LibraryPatch.from_json(patch))

        if "groups" in json_value:
            library._groups = json_value["groups"]

        return library
Example #4
0
 def __init__(self,
              operation: LibraryOperation,
              sdk_version: LibraryVersion,
              library: Optional[Library] = None):
     self._operation = operation
     self._sdk_version = sdk_version
     self._library: Library = library or Library()
def library_from_example(example: Example) -> Library:
    return Library(
        sources=example.get_prop(ExampleProperty.SOURCES),
        includes=Property(public=example.get_prop(ExampleProperty.INCLUDES)),
        cflags=Property(public=example.get_prop(ExampleProperty.CFLAGS)),
        asmflags=Property(public=example.get_prop(ExampleProperty.ASMFLAGS)),
        ldflags=Property(public=example.get_prop(ExampleProperty.LDFLAGS)))
Example #6
0
def generate_libraries_dependencies(
        examples: List[Example],
        lib_descs: Dict[str, LibraryDescription]):

    print("Generating dependencies...")
    remaining_deps = len(lib_descs)

    for lib_desc_name in lib_descs:
        lib_desc = lib_descs[lib_desc_name]

        # Get info about dependencies generation
        sys.stdout.write("\rRemaining dependencies: " +
                         str(remaining_deps).ljust(4)
                         )
        remaining_deps -= 1

        # Function which checks if all sources from the library are included
        # in the example sources

        def includes_all_sources_from_library(example: Example):
            example_sources = example.get_prop(ExampleProperty.SOURCES)
            library_sources = lib_desc.library.sources
            sources_intersection = set.intersection(
                example_sources,
                library_sources
            )
            return len(sources_intersection) == len(library_sources)

        # Find library converted from all examples, which include all sources
        # from the library.
        intersection_lib = Library.intersection(
            library_from_example(example) for example in examples
            if includes_all_sources_from_library(example)
        )

        # Remove used resources from library
        intersection_lib.difference_update(lib_desc.library)

        # While there are any sources left we try to find library for them.
        dependencies = Property()
        while len(intersection_lib.sources) > 0:

            # Take one random source and find library for it.
            source = intersection_lib.sources.pop()
            found_library = find_library_including_source(source, lib_descs)

            if found_library == None:
                print("WARNING: library not found for: " + source)
            else:
                dependencies.add_item(found_library[0], Access.PUBLIC)
                intersection_lib.difference_update(found_library[1].library)

        # Update library's dependencies:
        lib_desc.library.set_prop(LibraryProperty.DEPENDENCIES, dependencies)

    # Info about success
    print("\nDependency generation finished!")
    def test_library_patches(self):
        lib_desc = LibraryDescription(
            variant=LibraryVariant.OBJECT,
            documentation="",
            sdk_version=LibraryVersion(Version(15, 0, 0), Version(16, 0, 0)),
            library=Library(sources={"s1", "s2"}),
            patches=[
                LibraryPatch(LibraryOperation.ADD,
                             LibraryVersion(Version(15, 1, 0)),
                             Library({"s3"})),
                LibraryPatch(
                    LibraryOperation.ADD,
                    LibraryVersion(Version(15, 2, 0), Version(15, 3, 0)),
                    Library({"s4"})),
                LibraryPatch(
                    LibraryOperation.REMOVE,
                    LibraryVersion(Version(15, 2, 1), Version(15, 4, 0)),
                    Library({"s1"}))
            ])

        self.assertEqual(lib_desc.library_for_sdk_version(Version(14, 0, 0)),
                         None)
        self.assertEqual(lib_desc.library_for_sdk_version(Version(16, 1, 0)),
                         None)
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 0, 2)).sources,
            {'s1', 's2'})
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 1, 2)).sources,
            {'s1', 's2', 's3'})
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 2, 0)).sources,
            {'s1', 's2', 's3', 's4'})
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 2, 2)).sources,
            {'s2', 's3', 's4'})
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 4, 0)).sources,
            {'s2', 's3'})
        self.assertSetEqual(
            lib_desc.library_for_sdk_version(Version(15, 4, 1)).sources,
            {'s1', 's2', 's3'})
Example #8
0
    def from_json(json_value: dict) -> LibraryPatch:
        validate_json(instance=json_value,
                      schema=LibraryPatch.json_schema)

        patch = LibraryPatch(
            LibraryOperation(json_value["operation"]),
            LibraryVersion.from_json(json_value["sdk_version"])
        )
        patch._library = Library.from_json(json_value)

        return patch
 def __init__(self,
              variant: LibraryVariant = LibraryVariant.OBJECT,
              documentation: str = "",
              sdk_version: Optional[LibraryVersion] = None,
              library: Optional[Library] = None,
              patches: Optional[List[LibraryPatch]] = None):
     self._variant = variant
     self._documentation = documentation
     self._sdk_version = sdk_version
     self._library: Library = library or Library()
     self._patches: List[LibraryPatch] = patches or []
    def library_for_sdk_version(self, version: Version) -> Optional[Library]:
        if not self.applies_to_sdk_version(version):
            return None

        result_library = Library()
        result_library.union_update(self.library)

        for patch in self.patches:
            if not patch.sdk_version.applies_to(version):
                continue

            if patch.operation == LibraryOperation.ADD:
                result_library.union_update(patch.library)
            else:
                result_library.difference_update(patch.library)

        return result_library
 def union_library() -> Library:
     return Library.union((library_from_example(x) for x in examples))
 def intersection_library_including_src(source: str) -> Library:
     return Library.intersection(
         library_from_example(e) for e in examples
         if source in e.get_prop(ExampleProperty.SOURCES))
 def remove_duplicates(library: Library):
     for lib in libraries_iter():
         library.difference_update(lib[1])
def generate_libraries(
    examples: Iterable[Example], known_libraries: Dict[str, LibraryDescription]
) -> Dict[str, LibraryDescription]:
    # Print what we are doing...
    print("Generating modules...")

    # This dict contains new, generated libraries
    gen_libraries: Dict[str, LibraryDescription] = {}
    gen_library_count: int = 0

    # All defined libraries iterator
    def libraries_iter():
        for known_lib in known_libraries.items():
            yield (known_lib[0], known_lib[1].library)
        for gen_lib in gen_libraries.items():
            yield (gen_lib[0], gen_lib[1].library)

    # Remove duplicates from a library which exists in already known and
    # generated libraries.
    def remove_duplicates(library: Library):
        for lib in libraries_iter():
            library.difference_update(lib[1])

    # This function finds intersection library from all examples which include
    # source file.
    def intersection_library_including_src(source: str) -> Library:
        return Library.intersection(
            library_from_example(e) for e in examples
            if source in e.get_prop(ExampleProperty.SOURCES))

    # This function creates union library from all examples
    def union_library() -> Library:
        return Library.union((library_from_example(x) for x in examples))

    # Collect union library from all examples and remove entires already present
    # in other libraries.
    union_lib = union_library()
    remove_duplicates(union_lib)

    # Iterate until all sources are processed from the union library
    while len(union_lib.sources) > 0:

        # Show progress...
        sys.stdout.write("\rRemaining sources: " +
                         str(len(union_lib.sources)).ljust(4))

        # Pick first source at random
        union_lib_src = next(iter(union_lib.sources))

        #  Get intersection library from all libraries which include union_lib_src
        library = intersection_library_including_src(union_lib_src)
        remove_duplicates(library)

        # Check if every source in the above library always appears in every
        # known example. That would suggest sources are part of the same lib.
        smaller_lib_found = True
        while smaller_lib_found:
            # Let's assume we didn't find smaller library...
            smaller_lib_found = False

            # Iterate all library's sources.
            for lib_src in library.sources:

                # Find intersection library from all examples which includes lib_src
                other_library = intersection_library_including_src(lib_src)
                remove_duplicates(other_library)

                # Intersect two libraries and check if sources remain the same.
                merged_library = Library.intersection([library, other_library])
                if len(merged_library.sources) != len(library.sources):
                    smaller_lib_found = True
                    library = merged_library
                    break

        # Name library and add it to generated libraries
        gen_libraries["lib" + str(gen_library_count)] = \
            LibraryDescription(library=library)
        gen_library_count += 1

        # Removed used sources and other props from union_lib
        union_lib.difference_update(library)

    # Validate all libraries by checking that all sources, includes etc. are
    # covered by gathered libs. These are only warnings.
    union_lib_from_examples = union_library()
    union_lib_from_libs = Library.union(x[1] for x in libraries_iter())

    # Check differences
    print("\nFinished processing all sources!")

    union_lib_from_examples.difference_update(union_lib_from_libs)
    if len(union_lib_from_examples.sources) > 0:
        print("WARNING: sources not handled:")
        pp.pprint(union_lib_from_examples.sources)

    for prop in LibraryProperty:
        all_items = union_lib_from_examples.get_prop(prop).get_all_items()
        if len(all_items) > 0:
            print("WARNING: " + prop.value + " not handled:")
            pp.pprint(all_items)

    # Return generated libs
    return gen_libraries