示例#1
0
    def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
        pomMain = True
        # The pom is main only if no other main artifact is available
        if len(extsAndClass) > 1 and self._containsMainArtifact(extsAndClass) and "pom" in extsAndClass:
            pomMain = False

        artTypes = []
        for ext, classifiers in extsAndClass.iteritems():
            main = ext == "pom" and pomMain
            if not main:
                for classifier in classifiers:
                    extClassifier = "%s:%s" % (ext, classifier or "")
                    main = extClassifier not in self.notMainExtClassifiers
                    if main:
                        break
            artTypes.append(ArtifactType(ext, main, classifiers))

        mavenArtifact = MavenArtifact(groupId, artifactId, None, version)
        if suffix is not None:
            mavenArtifact.snapshotVersionSuffix = suffix
        if mavenArtifact in artifacts:
            artifacts[mavenArtifact].merge(ArtifactSpec(url, artTypes))
        else:
            logging.debug("Adding artifact %s", str(mavenArtifact))
            artifacts[mavenArtifact] = ArtifactSpec(url, artTypes)
示例#2
0
    def test_maven_artifact(self):
        artifact1 = MavenArtifact.createFromGAV("org.jboss:jboss-parent:pom:10")
        self.assertEqual(artifact1.groupId, "org.jboss")
        self.assertEqual(artifact1.artifactId, "jboss-parent")
        self.assertEqual(artifact1.version, "10")
        self.assertEqual(artifact1.getArtifactType(), "pom")
        self.assertEqual(artifact1.getClassifier(), "")
        self.assertEqual(artifact1.getArtifactFilename(), "jboss-parent-10.pom")
        self.assertEqual(artifact1.getArtifactFilepath(), "org/jboss/jboss-parent/10/jboss-parent-10.pom")

        artifact2 = MavenArtifact.createFromGAV("org.jboss:jboss-foo:jar:1.0")
        self.assertEqual(artifact2.getArtifactFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.jar")
        self.assertEqual(artifact2.getPomFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.pom")
        self.assertEqual(artifact2.getSourcesFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0-sources.jar")

        artifact3 = MavenArtifact.createFromGAV("org.jboss:jboss-test:jar:client:2.0.0.Beta1")
        self.assertEqual(artifact3.getClassifier(), "client")
        self.assertEqual(artifact3.getArtifactFilename(), "jboss-test-2.0.0.Beta1-client.jar")
        self.assertEqual(artifact3.getArtifactFilepath(),
                "org/jboss/jboss-test/2.0.0.Beta1/jboss-test-2.0.0.Beta1-client.jar")

        artifact4 = MavenArtifact.createFromGAV("org.acme:jboss-bar:jar:1.0-alpha-1:compile")
        self.assertEqual(artifact4.getArtifactFilepath(), "org/acme/jboss-bar/1.0-alpha-1/jboss-bar-1.0-alpha-1.jar")

        artifact5 = MavenArtifact.createFromGAV("com.google.guava:guava:pom:r05")
        self.assertEqual(artifact5.groupId, "com.google.guava")
        self.assertEqual(artifact5.artifactId, "guava")
        self.assertEqual(artifact5.version, "r05")
        self.assertEqual(artifact5.getArtifactType(), "pom")
        self.assertEqual(artifact5.getClassifier(), "")
        self.assertEqual(artifact5.getArtifactFilename(), "guava-r05.pom")
    def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
        pomMain = True
        # The pom is main only if no other main artifact is available
        if len(extsAndClass) > 1 and self._containsMainArtifact(extsAndClass) and "pom" in extsAndClass:
            pomMain = False

        artTypes = []
        for ext, classifiers in extsAndClass.iteritems():
            main = ext == "pom" and pomMain
            if not main:
                for classifier in classifiers:
                    extClassifier = "%s:%s" % (ext, classifier or "")
                    main = extClassifier not in self.notMainExtClassifiers
                    if main:
                        break
            artTypes.append(ArtifactType(ext, main, classifiers))

        mavenArtifact = MavenArtifact(groupId, artifactId, None, version)
        if suffix is not None:
            mavenArtifact.snapshotVersionSuffix = suffix
        if mavenArtifact in artifacts:
            artifacts[mavenArtifact].merge(ArtifactSpec(url, artTypes))
        else:
            logging.debug("Adding artifact %s", str(mavenArtifact))
            artifacts[mavenArtifact] = ArtifactSpec(url, artTypes)
 def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
     if len(extsAndClass) > 1 and self._containsNonPomWithoutClassifier(extsAndClass) and "pom" in extsAndClass:
         del extsAndClass["pom"]
     for ext in extsAndClass:
         mavenArtifact = MavenArtifact(groupId, artifactId, ext, version)
         if suffix is not None:
             mavenArtifact.snapshotVersionSuffix = suffix
         logging.debug("Adding artifact %s", str(mavenArtifact))
         artifacts[mavenArtifact] = ArtifactSpec(url, extsAndClass[ext])
    def _filterExcludedTypes(self, artifactList):
        '''
        Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
        not-excluded artifact types only.

        :param artifactList: artifactList to be filtered.
        :param exclTypes: list of excluded types
        :returns: artifactList without artifacts that matched specified types and had no other main types.
        '''
        logging.debug("Filtering artifacts with excluded types.")
        regExps = maven_repo_util.getRegExpsFromStrings(self.config.gatcvWhitelist)
        exclTypes = self.config.excludedTypes
        for ga in artifactList.keys():
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artSpec = artifactList[ga][priority][version]
                    for artType in list(artSpec.artTypes.keys()):
                        if artType in exclTypes:
                            artTypeObj = artSpec.artTypes[artType]
                            classifiers = artTypeObj.classifiers
                            (groupId, artifactId) = ga.split(':')
                            for classifier in list(classifiers):
                                art = MavenArtifact(groupId, artifactId, artType, version, classifier)
                                gatcv = art.getGATCV()
                                if not maven_repo_util.somethingMatch(regExps, gatcv):
                                    logging.debug("Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
                                                  "excluded type.", classifier, ga, artType, version, priority)
                                    classifiers.remove(classifier)
                                else:
                                    logging.debug("Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
                                                  "whitelist pattern.", ga, artType, classifier, version, priority)
                            if not classifiers:
                                logging.debug("Dropping %s:%s:%s from priority %i because of no classifier left.", ga,
                                              artType, version, priority)
                                del(artSpec.artTypes[artType])
                    noMain = True
                    for artType in artSpec.artTypes.keys():
                        artTypeObj = artSpec.artTypes[artType]
                        if artTypeObj.mainType:
                            noMain = False
                            break
                    if not artSpec.artTypes or noMain:
                        if noMain:
                            logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.",
                                          ga, version, priority)
                        else:
                            logging.debug("Dropping GAV %s:%s from priority %i because of no artifact type left.",
                                          ga, version, priority)
                        del artifactList[ga][priority][version]
                if not artifactList[ga][priority]:
                    logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
                    del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.", ga)
                del artifactList[ga]
        return artifactList
 def _addArtifact(self, artifacts, groupId, artifactId, version,
                  extsAndClass, suffix, url):
     if len(extsAndClass) > 1 and self._containsNonPomWithoutClassifier(
             extsAndClass) and "pom" in extsAndClass:
         del extsAndClass["pom"]
     for ext in extsAndClass:
         mavenArtifact = MavenArtifact(groupId, artifactId, ext, version)
         if suffix is not None:
             mavenArtifact.snapshotVersionSuffix = suffix
         logging.debug("Adding artifact %s", str(mavenArtifact))
         artifacts[mavenArtifact] = ArtifactSpec(url, extsAndClass[ext])
示例#7
0
    def test_listDependencies(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavs = [
            'com.sun.faces:jsf-api:2.0.11',
            'org.apache.ant:ant:1.8.0'
        ]
        dependencies = {
            'javax.servlet:javax.servlet-api:jar:3.0.1': set(['javadoc', 'sources']),
            'javax.servlet.jsp.jstl:jstl-api:jar:1.2': set(['javadoc', 'sources']),
            'xml-apis:xml-apis:jar:1.3.04': set(['source', 'sources']),
            'javax.servlet:servlet-api:jar:2.5': set(['sources']),
            'javax.el:javax.el-api:jar:2.2.1': set(['javadoc', 'sources']),
            'junit:junit:jar:3.8.2': set(['javadoc', 'sources']),
            'xerces:xercesImpl:jar:2.9.0': set([]),
            'javax.servlet.jsp:jsp-api:jar:2.1': set(['sources']),
            'javax.servlet.jsp:javax.servlet.jsp-api:jar:2.2.1': set(['javadoc', 'sources']),
            'org.apache.ant:ant-launcher:jar:1.8.0': set([])
        }
        expectedArtifacts = {}
        for dep in dependencies:
            artifact = MavenArtifact.createFromGAV(dep)
            expectedArtifacts[artifact] = ArtifactSpec(repoUrls[0], dependencies[dep])

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listDependencies(repoUrls, gavs, False, False)

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#8
0
def generateArtifactList(options):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L artifacts (list of MavenArtifact)
    """

    options.allclassifiers = (options.classifiers == '__all__')

    artifactList = _generateArtifactList(options)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for gat in artifactList:
        priorityList = artifactList[gat]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for classifier in artSpec.classifiers:
                    if classifier == "" or options.allclassifiers:
                        artifact = MavenArtifact.createFromGAV(gat + (
                            (":" + classifier) if classifier else "") + ":" +
                                                               version)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList
def generateArtifactList(options, args):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L list of MavenArtifact
    """

    artifactList = _generateArtifactList(options, args)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for ga in artifactList:
        priorityList = artifactList[ga]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for artType in artSpec.artTypes.keys():
                    for classifier in artSpec.artTypes[artType].classifiers:
                        if classifier:
                            gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, version)
                        else:
                            gatcv = "%s:%s:%s" % (ga, artType, version)
                        artifact = MavenArtifact.createFromGAV(gatcv)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList
def generateArtifactList(options):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L artifacts (list of MavenArtifact)
    """

    options.allclassifiers = (options.classifiers == '__all__')

    artifactList = _generateArtifactList(options)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for gat in artifactList:
        priorityList = artifactList[gat]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for classifier in artSpec.classifiers:
                    if classifier == "" or options.allclassifiers:
                        artifact = MavenArtifact.createFromGAV(gat + ((":" + classifier) if classifier else "") +
                                                               ":" + version)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList
def generateArtifactList(options, args):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L list of MavenArtifact
    """

    artifactList = _generateArtifactList(options, args)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for ga in artifactList:
        priorityList = artifactList[ga]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                artSpec = versionList[version]
                url = artSpec.url
                for artType in artSpec.artTypes.keys():
                    for classifier in artSpec.artTypes[artType].classifiers:
                        if classifier:
                            gatcv = "%s:%s:%s:%s" % (ga, artType, classifier,
                                                     version)
                        else:
                            gatcv = "%s:%s:%s" % (ga, artType, version)
                        artifact = MavenArtifact.createFromGAV(gatcv)
                        urlToMAList.setdefault(url, []).append(artifact)
    return urlToMAList
示例#12
0
    def test_listRepository_file(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['file://./tests/testrepo']
        gavPatterns = ['bar:foo-bar:1.1', 'foo.baz:baz-core:1.0']

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]):
            ArtifactSpec(repoUrls[0], set([])),
            MavenArtifact.createFromGAV(gavPatterns[1]):
            ArtifactSpec(repoUrls[0], set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
    def _filterExcludedRepositories(self, artifactList, threadnum):
        """
        Filter artifactList removing artifacts existing in specified repositories.

        :param artifactList: artifactList to be filtered.
        :returns: artifactList without artifacts that exists in specified repositories.
        """

        logging.debug(
            "Filtering artifacts contained in excluded repositories.")

        pool = ThreadPool(threadnum)
        # Contains artifact to be removed
        delArtifacts = []
        for ga in artifactList.keys():
            groupId = ga.split(':')[0]
            artifactId = ga.split(':')[1]
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artifact = MavenArtifact(groupId, artifactId, "pom",
                                             version)
                    pool.apply_async(_artifactInRepos, [
                        self.config.excludedRepositories, artifact, priority,
                        delArtifacts
                    ])

        # Close the pool and wait for the workers to finnish
        pool.close()
        pool.join()
        for artifact, priority in delArtifacts:
            ga = artifact.getGA()
            logging.debug(
                "Dropping GAV %s:%s from priority %i because it was found in an excluded repository.",
                ga, artifact.version, priority)
            del artifactList[ga][priority][artifact.version]
            if not artifactList[ga][priority]:
                logging.debug(
                    "Dropping GA %s from priority %i because of no version left.",
                    ga, priority)
                del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.",
                              ga)
                del artifactList[ga]

        return artifactList
示例#14
0
    def test_listRepository_http(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavPatterns = [
            'com.sun.faces:jsf-api:2.0.11',
            'org.apache.ant:ant:1.8.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources'])),
            MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set([]))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#15
0
    def test_listRepository_file(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['file://./tests/testrepo']
        gavPatterns = [
            'bar:foo-bar:1.1',
            'foo.baz:baz-core:1.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set([])),
            MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#16
0
    def test_maven_artifact(self):
        artifact1 = MavenArtifact.createFromGAV(
            "org.jboss:jboss-parent:pom:10")
        self.assertEqual(artifact1.groupId, "org.jboss")
        self.assertEqual(artifact1.artifactId, "jboss-parent")
        self.assertEqual(artifact1.version, "10")
        self.assertEqual(artifact1.getArtifactType(), "pom")
        self.assertEqual(artifact1.getClassifier(), "")
        self.assertEqual(artifact1.getArtifactFilename(),
                         "jboss-parent-10.pom")
        self.assertEqual(artifact1.getArtifactFilepath(),
                         "org/jboss/jboss-parent/10/jboss-parent-10.pom")

        artifact2 = MavenArtifact.createFromGAV("org.jboss:jboss-foo:jar:1.0")
        self.assertEqual(artifact2.getArtifactFilepath(),
                         "org/jboss/jboss-foo/1.0/jboss-foo-1.0.jar")
        self.assertEqual(artifact2.getPomFilepath(),
                         "org/jboss/jboss-foo/1.0/jboss-foo-1.0.pom")
        self.assertEqual(artifact2.getSourcesFilepath(),
                         "org/jboss/jboss-foo/1.0/jboss-foo-1.0-sources.jar")

        artifact3 = MavenArtifact.createFromGAV(
            "org.jboss:jboss-test:jar:client:2.0.0.Beta1")
        self.assertEqual(artifact3.getClassifier(), "client")
        self.assertEqual(artifact3.getArtifactFilename(),
                         "jboss-test-2.0.0.Beta1-client.jar")
        self.assertEqual(
            artifact3.getArtifactFilepath(),
            "org/jboss/jboss-test/2.0.0.Beta1/jboss-test-2.0.0.Beta1-client.jar"
        )

        artifact4 = MavenArtifact.createFromGAV(
            "org.acme:jboss-bar:jar:1.0-alpha-1:compile")
        self.assertEqual(
            artifact4.getArtifactFilepath(),
            "org/acme/jboss-bar/1.0-alpha-1/jboss-bar-1.0-alpha-1.jar")

        artifact5 = MavenArtifact.createFromGAV(
            "com.google.guava:guava:pom:r05")
        self.assertEqual(artifact5.groupId, "com.google.guava")
        self.assertEqual(artifact5.artifactId, "guava")
        self.assertEqual(artifact5.version, "r05")
        self.assertEqual(artifact5.getArtifactType(), "pom")
        self.assertEqual(artifact5.getClassifier(), "")
        self.assertEqual(artifact5.getArtifactFilename(), "guava-r05.pom")
        def findArtifact(gav, urls, artifacts):
            artifact = MavenArtifact.createFromGAV(gav)
            for url in urls:
                if maven_repo_util.gavExists(url, artifact):
                    #Critical section?
                    artifacts[artifact] = ArtifactSpec(url, [ArtifactType(artifact.artifactType, True, set(['']))])
                    return

            logging.warning('Artifact %s not found in any url!', artifact)
示例#18
0
    def test_listRepository_file_gatcvs(self):
        config = configuration.Configuration()
        config.addClassifiers = "__all__"
        repoUrls = ['file://./tests/testrepo']
        gatcvs = [
            'bar:foo-bar:pom:1.1',
            'foo.baz:baz-core:jar:1.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, None, gatcvs)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gatcvs[0]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set(['']))]),
            MavenArtifact.createFromGAV(gatcvs[1]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set([''])),
                                        ArtifactType("jar", True, set(['', 'javadoc', 'sources']))])
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#19
0
    def test_listRepository_http(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavPatterns = [
            'com.sun.faces:jsf-api:2.0.11', 'org.apache.ant:ant:1.8.0'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]):
            ArtifactSpec(repoUrls[0], set(['javadoc', 'sources'])),
            MavenArtifact.createFromGAV(gavPatterns[1]):
            ArtifactSpec(repoUrls[0], set([]))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
        def findArtifact(gav, urls, artifacts):
            artifact = MavenArtifact.createFromGAV(gav)
            for url in urls:
                if maven_repo_util.gavExists(url, artifact):
                    #Critical section?
                    artifacts[artifact] = ArtifactSpec(url)
                    return

            logging.warning('Artifact %s not found in any url!', artifact)
示例#21
0
    def test_listRepository_file_gatcvs(self):
        config = configuration.Configuration()
        config.addClassifiers = "__all__"
        repoUrls = ['file://./tests/testrepo']
        gatcvs = ['bar:foo-bar:pom:1.1', 'foo.baz:baz-core:jar:1.0']

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listRepository(repoUrls, None, gatcvs)
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gatcvs[0]):
            ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set(['']))]),
            MavenArtifact.createFromGAV(gatcvs[1]):
            ArtifactSpec(repoUrls[0], [
                ArtifactType("pom", True, set([''])),
                ArtifactType("jar", True, set(['', 'javadoc', 'sources']))
            ])
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#22
0
    def _getExpectedArtifacts(self, repoUrl, dependencies):

        artSpecDict = {}
        for dep in dependencies.keys():
            artifact = MavenArtifact.createFromGAV(dep)
            artTypes = []
            artTypes.append(ArtifactType(artifact.artifactType, artifact.artifactType != "pom", dependencies[dep]))
            artSpec = ArtifactSpec(repoUrl, artTypes)
            gav = artifact.getGAV()
            if gav in artSpecDict.keys():
                artSpecDict[gav].merge(artSpec)
            else:
                artSpecDict[gav] = artSpec
        expectedArtifacts = {}
        for (gav, artSpec) in artSpecDict.iteritems():
            artifact = MavenArtifact.createFromGAV(gav)
            expectedArtifacts[artifact] = artSpec

        return expectedArtifacts
示例#23
0
def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = maven_repo_util.parseGATCVS(nextLine)
        if gav:
            artifactList.append(MavenArtifact.createFromGAV(gav))
    return artifactList
示例#24
0
    def _filterExcludedRepositories(self, artifactList):
        """
        Filter artifactList removing artifacts existing in specified repositories.

        :param artifactList: artifactList to be filtered.
        :returns: artifactList without artifacts that exists in specified repositories.
        """

        logging.debug("Filtering artifacts contained in excluded repositories.")

        pool = ThreadPool(maven_repo_util.MAX_THREADS)
        # Contains artifact to be removed
        delArtifacts = []
        for ga in artifactList.keys():
            groupId = ga.split(':')[0]
            artifactId = ga.split(':')[1]
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artifact = MavenArtifact(groupId, artifactId, "pom", version)
                    pool.apply_async(
                        _artifactInRepos,
                        [self.config.excludedRepositories, artifact, priority, delArtifacts]
                    )

        # Close the pool and wait for the workers to finnish
        pool.close()
        pool.join()
        for artifact, priority in delArtifacts:
            ga = artifact.getGA()
            logging.debug("Dropping GAV %s:%s from priority %i because it was found in an excluded repository.",
                          ga, artifact.version, priority)
            del artifactList[ga][priority][artifact.version]
            if not artifactList[ga][priority]:
                logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
                del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.", ga)
                del artifactList[ga]

        return artifactList
示例#25
0
    def test_listDependencies_recursive(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        repoUrls = ['http://repo.maven.apache.org/maven2/']
        gavs = ['com.sun.faces:jsf-api:2.0.11', 'org.apache.ant:ant:1.8.0']
        dependencies = {
            'junit:junit:jar:3.8.2':
            set(['sources', 'javadoc']),
            'junit:junit:pom:3.8.2':
            set(['sources', 'javadoc']),
            'xerces:xercesImpl:jar:2.9.0':
            set([]),
            'xml-apis:xml-apis:jar:1.3.04':
            set(['source', 'sources']),
            'xml-apis:xml-apis:pom:1.3.04':
            set(['source', 'sources']),
            'javax.el:javax.el-api:jar:2.2.1':
            set(['sources', 'javadoc']),
            'javax.el:javax.el-api:pom:2.2.1':
            set(['sources', 'javadoc']),
            'xml-resolver:xml-resolver:jar:1.2':
            set(['sources']),
            'javax.servlet:servlet-api:jar:2.5':
            set(['sources']),
            'javax.servlet:servlet-api:pom:2.5':
            set(['sources']),
            'javax.servlet.jsp:jsp-api:jar:2.1':
            set(['sources']),
            'javax.servlet.jsp:jsp-api:pom:2.1':
            set(['sources']),
            'org.apache.ant:ant-launcher:jar:1.8.0':
            set([]),
            'javax.servlet.jsp.jstl:jstl-api:jar:1.2':
            set(['sources', 'javadoc']),
            'javax.servlet:javax.servlet-api:jar:3.0.1':
            set(['sources', 'javadoc']),
            'javax.servlet:javax.servlet-api:pom:3.0.1':
            set(['sources', 'javadoc']),
            'javax.servlet.jsp:javax.servlet.jsp-api:jar:2.2.1':
            set(['sources', 'javadoc'])
        }
        expectedArtifacts = {}
        for dep in dependencies:
            artifact = MavenArtifact.createFromGAV(dep)
            expectedArtifacts[artifact] = ArtifactSpec(repoUrls[0],
                                                       dependencies[dep])

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listDependencies(repoUrls, gavs, True,
                                                    False)

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#26
0
    def _getExpectedArtifacts(self, repoUrl, dependencies):

        artSpecDict = {}
        for dep in dependencies.keys():
            artifact = MavenArtifact.createFromGAV(dep)
            artTypes = []
            artTypes.append(
                ArtifactType(artifact.artifactType,
                             artifact.artifactType != "pom",
                             dependencies[dep]))
            artSpec = ArtifactSpec(repoUrl, artTypes)
            gav = artifact.getGAV()
            if gav in artSpecDict.keys():
                artSpecDict[gav].merge(artSpec)
            else:
                artSpecDict[gav] = artSpec
        expectedArtifacts = {}
        for (gav, artSpec) in artSpecDict.iteritems():
            artifact = MavenArtifact.createFromGAV(gav)
            expectedArtifacts[artifact] = artSpec

        return expectedArtifacts
def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    regexGAV = re.compile('(([\w\-.]+:){3}[\w\-.]+)(:[\w]*\S)?')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = regexGAV.search(nextLine)
        if gav:
            artifactList.append(MavenArtifact(gav.group(0)))
    return artifactList
def generate_report(output, config, artifact_list, report_name):
    """
    Generates report. The report consists of a summary page, groupId pages, artifactId pages and leaf artifact pages.
    Summary page contains list of roots, list of BOMs, list of multi-version artifacts, links to all artifacts and list
    of artifacts that do not match the BOM version. Each artifact has a separate page containing paths from roots to
    the artifact with path explanation.
    """
    multiversion_gas = dict()
    malformed_versions = dict()
    if os.path.exists(output):
        logging.warn("Target report path %s exists. Deleting...", output)
        shutil.rmtree(output)
    os.makedirs(os.path.join(output, "pages"))

    roots = []
    for artifact_source in config.artifactSources:
        if artifact_source["type"] == "dependency-graph":
            roots.extend(artifact_source["top-level-gavs"])

    groupids = dict()
    version_pattern = re.compile("^.*[.-]redhat-[^.]+$")
    for ga in artifact_list:
        (groupid, artifactid) = ga.split(":")
        priority_list = artifact_list[ga]
        for priority in priority_list:
            versions = priority_list[priority]
            if versions:
                groupids.setdefault(groupid, dict()).setdefault(artifactid, dict()).update(versions)
                if len(groupids[groupid][artifactid]) > 1:
                    multiversion_gas.setdefault(groupid, dict())[artifactid] = groupids[groupid][artifactid]
                for version in versions:
                    if not version_pattern.match(version):
                        malformed_versions.setdefault(groupid, dict()).setdefault(artifactid, dict())[
                            version
                        ] = groupids[groupid][artifactid][version]

    optional_artifacts = dict()
    for groupid in groupids.keys():
        artifactids = groupids[groupid]
        for artifactid in artifactids.keys():
            versions = artifactids[artifactid]
            for version in versions.keys():
                art_spec = versions[version]

                ma = MavenArtifact.createFromGAV("%s:%s:%s" % (groupid, artifactid, version))
                generate_artifact_page(ma, roots, art_spec.paths, art_spec.url, output, groupids, optional_artifacts)
            generate_artifactid_page(groupid, artifactid, versions, output)
        generate_groupid_page(groupid, artifactids, output)
    generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts)
    generate_css(output)
def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    # Match pattern groupId:artifactId:type:[classifier:]version[:scope]
    regexGAV = re.compile('(([\w\-.]+:){3}([\w\-.]+:)?([\d][\w\-.]+))(:[\w]*\S)?')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = regexGAV.search(nextLine)
        if gav:
            artifactList.append(MavenArtifact.createFromGAV(gav.group(1)))
    return artifactList
    def _filterExcludedRepositories(self, artifactList):
        """
        Filter artifactList removing artifacts existing in specified repositories.

        :param artifactList: artifactList to be filtered.
        :returns: artifactList without arifacts that exists in specified repositories.
        """

        logging.debug("Filtering artifacts contained in excluded repositories.")

        pool = ThreadPool(maven_repo_util.MAX_THREADS)
        # Contains artifact to be removed
        delArtifacts = []
        for gat in artifactList.keys():
            groupId = gat.split(':')[0]
            artifactId = gat.split(':')[1]
            artifactType = gat.split(':')[2]
            for priority in artifactList[gat].keys():
                for version in artifactList[gat][priority].keys():
                    artifact = MavenArtifact(groupId, artifactId, artifactType, version)
                    pool.apply_async(
                        _artifactInRepos,
                        [self.config.excludedRepositories, artifact, priority, delArtifacts]
                    )
                if not artifactList[gat][priority]:
                    del artifactList[gat][priority]
            if not artifactList[gat]:
                del artifactList[gat]

        # Close the pool and wait for the workers to finnish
        pool.close()
        pool.join()
        for artifact, priority in delArtifacts:
            del artifactList[artifact.getGAT()][priority][artifact.version]

        return artifactList
示例#31
0
def depListToArtifactList(depList):
    """Convert the maven GAV to a URL relative path"""
    regexComment = re.compile('#.*$')
    #regexLog = re.compile('^\[\w*\]')
    # Match pattern groupId:artifactId:type:[classifier:]version[:scope]
    regexGAV = re.compile(
        '(([\w\-.]+:){3}([\w\-.]+:)?([\d][\w\-.]+))(:[\w]*\S)?')
    artifactList = []
    for nextLine in depList:
        nextLine = regexComment.sub('', nextLine)
        nextLine = nextLine.strip()
        gav = regexGAV.search(nextLine)
        if gav:
            artifactList.append(MavenArtifact.createFromGAV(gav.group(1)))
    return artifactList
def generate_report(output, config, artifact_list, report_name):
    """
    Generates report. The report consists of a summary page, groupId pages, artifactId pages and leaf artifact pages.
    Summary page contains list of roots, list of BOMs, list of multi-version artifacts, links to all artifacts and list
    of artifacts that do not match the BOM version. Each artifact has a separate page containing paths from roots to
    the artifact with path explanation.
    """
    multiversion_gas = dict()
    malformed_versions = dict()
    if os.path.exists(output):
        logging.warn("Target report path %s exists. Deleting...", output)
        shutil.rmtree(output)
    os.makedirs(os.path.join(output, "pages"))

    roots = []
    for artifact_source in config.artifactSources:
        if artifact_source["type"] == "dependency-graph":
            roots.extend(artifact_source['top-level-gavs'])

    groupids = dict()
    version_pattern = re.compile("^.*[.-]redhat-[^.]+$")
    for ga in artifact_list:
        (groupid, artifactid) = ga.split(":")
        priority_list = artifact_list[ga]
        for priority in priority_list:
            versions = priority_list[priority]
            if versions:
                groupids.setdefault(groupid, dict()).setdefault(artifactid, dict()).update(versions)
                if len(groupids[groupid][artifactid]) > 1:
                    multiversion_gas.setdefault(groupid, dict())[artifactid] = groupids[groupid][artifactid]
                for version in versions:
                    if not version_pattern.match(version):
                        malformed_versions.setdefault(groupid, dict()).setdefault(artifactid, dict())[version] = groupids[groupid][artifactid][version]

    optional_artifacts = dict()
    for groupid in groupids.keys():
        artifactids = groupids[groupid]
        for artifactid in artifactids.keys():
            versions = artifactids[artifactid]
            for version in versions.keys():
                art_spec = versions[version]

                ma = MavenArtifact.createFromGAV("%s:%s:%s" % (groupid, artifactid, version))
                generate_artifact_page(ma, roots, art_spec.paths, art_spec.url, output, groupids, optional_artifacts)
            generate_artifactid_page(groupid, artifactid, versions, output)
        generate_groupid_page(groupid, artifactids, output)
    generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts)
    generate_css(output)
    def _listDependencyGraph(self, aproxUrl, wsid, sourceKey, gavs):
        """
        Loads maven artifacts from dependency graph.

        :param aproxUrl: URL of the AProx instance
        :param gavs: List of top level GAVs
        :param wsid: workspace ID
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        from aprox_apis import AproxApi10
        aprox = AproxApi10(aproxUrl)

        deleteWS = False

        if not wsid:
            # Create workspace
            ws = aprox.createWorkspace()
            wsid = ws["id"]
            deleteWS = True

        # Resolve graph MANIFEST for GAVs
        urlmap = aprox.urlmap(wsid, sourceKey, gavs,
                              self.configuration.allClassifiers)

        # parse returned map
        artifacts = {}
        for gav in urlmap:
            artifact = MavenArtifact.createFromGAV(gav)
            groupId = artifact.groupId
            artifactId = artifact.artifactId
            version = artifact.version

            filenames = urlmap[gav]["files"]
            url = urlmap[gav]["repoUrl"]

            (extsAndClass, suffix) = self._getExtensionsAndClassifiers(
                artifactId, version, filenames)

            self._addArtifact(artifacts, groupId, artifactId, version,
                              extsAndClass, suffix, url)

        # cleanup
        if deleteWS:
            aprox.deleteWorkspace(wsid)

        return artifacts
    def _listDependencies(self, repoUrls, gavs):
        """
        Loads maven artifacts from mvn dependency:list.

        :param repoUrls: URL of the repositories that contains the listed artifacts
        :param gavs: List of top level GAVs
        :returns: Dictionary where index is MavenArtifact object and value is
                  it's repo root URL, or empty dictionary if something goes wrong.
        """
        artifacts = {}

        for gav in gavs:
            artifact = MavenArtifact.createFromGAV(gav)

            pomDir = 'poms'
            fetched = False
            for repoUrl in repoUrls:
                pomUrl = repoUrl + '/' + artifact.getPomFilepath()
                if fetchArtifact(pomUrl, pomDir):
                    fetched = True
                    break

            if not fetched:
                logging.warning("Failed to retrieve pom file for artifact %s",
                                gav)
                continue

            # Build dependency:list
            mvnOutDir = "maven"
            if not os.path.isdir(mvnOutDir):
                os.makedirs(mvnOutDir)
            mvnOutFilename = mvnOutDir + "/" + artifact.getBaseFilename() + "-maven.out"
            with open(mvnOutFilename, "w") as mvnOutputFile:
                retCode = call(['mvn', 'dependency:list', '-N', '-f',
                                pomDir + '/' + artifact.getPomFilename()], stdout=mvnOutputFile)

                if retCode != 0:
                    logging.warning("Maven failed to finish with success. Skipping artifact %s",
                                    gav)
                    continue

            # Parse GAVs from maven output
            gavList = self._parseDepList(mvnOutFilename)

            artifacts.update(self._listArtifacts(repoUrls, gavList))

        return artifacts
示例#35
0
    def test_listMeadTagArtifacts(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        kojiUrl = "http://brewhub.devel.redhat.com/brewhub/"
        tagName = "mead-import-maven"
        downloadRootUrl = "http://download.devel.redhat.com/brewroot/packages/"
        gavPatterns = ['org.apache.maven:maven-core:2.0.6']

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listMeadTagArtifacts(
            kojiUrl, downloadRootUrl, tagName, gavPatterns)
        expectedUrl = downloadRootUrl + "org.apache.maven-maven-core/2.0.6/1/maven/"
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]):
            ArtifactSpec(expectedUrl, set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
示例#36
0
    def test_listMeadTagArtifacts(self):
        config = configuration.Configuration()
        config.allClassifiers = True
        kojiUrl = "http://brewhub.devel.redhat.com/brewhub/"
        tagName = "mead-import-maven"
        downloadRootUrl = "http://download.devel.redhat.com/brewroot/packages/"
        gavPatterns = [
            'org.apache.maven:maven-core:2.0.6'
        ]

        builder = artifact_list_builder.ArtifactListBuilder(config)
        actualArtifacts = builder._listMeadTagArtifacts(kojiUrl, downloadRootUrl, tagName, gavPatterns)
        expectedUrl = downloadRootUrl + "org.apache.maven-maven-core/2.0.6/1/maven/"
        expectedArtifacts = {
            MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(expectedUrl, set(['javadoc', 'sources']))
        }

        self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
    def _listDependencyGraph(self, aproxUrl, wsid, sourceKey, gavs):
        """
        Loads maven artifacts from dependency graph.

        :param aproxUrl: URL of the AProx instance
        :param gavs: List of top level GAVs
        :param wsid: workspace ID
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        from aprox_apis import AproxApi10
        aprox = AproxApi10(aproxUrl)

        deleteWS = False

        if not wsid:
            # Create workspace
            ws = aprox.createWorkspace()
            wsid = ws["id"]
            deleteWS = True

        # Resolve graph MANIFEST for GAVs
        urlmap = aprox.urlmap(wsid, sourceKey, gavs, self.configuration.allClassifiers)

        # parse returned map
        artifacts = {}
        for gav in urlmap:
            artifact = MavenArtifact.createFromGAV(gav)
            groupId = artifact.groupId
            artifactId = artifact.artifactId
            version = artifact.version

            filenames = urlmap[gav]["files"]
            url = urlmap[gav]["repoUrl"]

            (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)

            self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)

        # cleanup
        if deleteWS:
            aprox.deleteWorkspace(wsid)

        return artifacts
def generateArtifactList(options):
    """
    Generates artifact "list" from sources defined in the given configuration in options. The result
    is dictionary with following structure:

    <repo url> (string)
      L artifacts (list of MavenArtifact)
    """

    artifactList = _generateArtifactList(options)
    #build sane structure - url to MavenArtifact list
    urlToMAList = {}
    for gat in artifactList:
        prioList = artifactList[gat]
        for priority in prioList:
            verList = prioList[priority]
            for version in verList:
                url = verList[version]
                urlToMAList.setdefault(url, []).append(MavenArtifact.createFromGAV(gat + ":" + version))

    return urlToMAList
    def _listArtifacts(self, urls, gavs):
        """
        Loads maven artifacts from list of GAVs and tries to locate the artifacts in one of the
        specified repositories.

        :param urls: repository URLs where the given GAVs can be located
        :param gavs: List of GAVs
        :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL.
        """
        artifacts = {}
        for gav in gavs:
            artifact = MavenArtifact.createFromGAV(gav)
            for url in urls:
                gavUrl = url + '/' + artifact.getDirPath()
                if mrbutils.urlExists(gavUrl):
                    artifacts[artifact] = url
                    break
            if not artifact in artifacts:
                logging.warning('artifact %s not found in any url!', artifact)

        return artifacts
    def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing):
        """
        Loads maven artifacts from mvn dependency:list.

        :param repoUrls: URL of the repositories that contains the listed artifacts
        :param gavs: List of top level GAVs
        :param recursive: runs dependency:list recursively using the previously discovered dependencies if True
        :param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None
                              to use Maven's default
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        artifacts = {}
        workingSet = set(gavs)
        checkedSet = set()

        while workingSet:
            gav = workingSet.pop()
            checkedSet.add(gav)
            logging.debug("Resolving dependencies for %s", gav)
            artifact = MavenArtifact.createFromGAV(gav)

            pomFilename = 'poms/' + artifact.getPomFilename()
            successPomUrl = None
            fetched = False
            for repoUrl in repoUrls:
                pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
                fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
                if fetched:
                    successPomUrl = repoUrl
                    break

            if not fetched:
                logging.warning("Failed to retrieve pom file for artifact %s", gav)
                continue

            tempDir = maven_repo_util.getTempDir()
            if not os.path.exists(tempDir):
                os.makedirs(tempDir)

            # Create settings.xml
            settingsFile = tempDir + "settings.xml"
            settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
                                               .replace('${temp}', maven_repo_util.getTempDir())
            with open(settingsFile, 'w') as settings:
                settings.write(settingsContent)

            # Build dependency:list
            depsDir = tempDir + "maven-deps-output/"
            outFile = depsDir + gav + ".out"
            args = ['mvn', 'dependency:list', '-N',
                                              '-DoutputFile=' + outFile,
                                              '-f', pomFilename,
                                              '-s', settingsFile]
            if include_scope:
                args.append("-DincludeScope=%s" % include_scope)
            logging.debug("Running Maven:\n  %s", " ".join(args))
            logging.debug("settings.xml contents: %s", settingsContent)
            mvn = Popen(args, stdout=PIPE)
            mvnStdout = mvn.communicate()[0]
            logging.debug("Maven output:\n%s", mvnStdout)

            if mvn.returncode != 0:
                logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
                continue

            with open(outFile, 'r') as out:
                depLines = out.readlines()
            gavList = self._parseDepList(depLines)
            logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))

            newArtifacts = self._listArtifacts(repoUrls, gavList)

            if recursive:
                for artifact in newArtifacts:
                    ngav = artifact.getGAV()
                    if ngav not in checkedSet:
                        workingSet.add(ngav)

            if self.configuration.isAllClassifiers():
                resultingArtifacts = {}
                for artifact in newArtifacts.keys():
                    spec = newArtifacts[artifact]
                    try:
                        out = self._lftpFind(spec.url + artifact.getDirPath())
                    except IOError as ex:
                        if skipmissing:
                            logging.warn("Error while listing files in %s: %s. Skipping...",
                                         spec.url + artifact.getDirPath(), str(ex))
                            continue
                        else:
                            raise ex

                    files = []
                    for line in out.split('\n'):
                        if line != "./" and line != "":
                            files.append(line[2:])

                    (extsAndClass, suffix) = self._getExtensionsAndClassifiers(
                        artifact.artifactId, artifact.version, files)
                    if artifact.artifactType in extsAndClass:
                        self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId,
                                          artifact.version, extsAndClass, suffix, spec.url)
                    else:
                        if files:
                            logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s",
                                         artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files))
                        else:
                            logging.warn("An empty filelist was listed from %s. Skipping...",
                                         spec.url + artifact.getDirPath())
                newArtifacts = resultingArtifacts

            artifacts.update(newArtifacts)

        return artifacts
def generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts):
    html = ("<html><head><title>Repository {report_name}</title>" + \
            "<link rel=\"stylesheet\" type=\"text/css\" href=\"http://code.jquery.com/ui/1.11.4/themes/smoothness/jquery-ui.css\">" + \
            "<script src=\"http://code.jquery.com/jquery-1.10.2.js\"></script>" + \
            "<script src=\"http://code.jquery.com/ui/1.11.4/jquery-ui.js\"></script>" + \
            "<link rel=\"stylesheet\" type=\"text/css\" href=\"pages/style.css\">" + \
            "<script>$(function() {script});</script></head><body>" + \
            "<div class=\"artifact\"><h1>{report_name}</h1><div id=\"tabs\">" + \
            "<ul>" + \
            "<li><a href=\"#tab-definition\">Artifact sources</a></li>" + \
            "<li><a href=\"#tab-multi-versioned-artifacts\">Multi-versioned artifacts</a></li>" + \
            "<li><a href=\"#tab-malformed-versions\">Malformed versions</a></li>" + \
            "<li><a href=\"#tab-optional-artifacts\">Optional artifacts</a></li>" + \
            "<li><a href=\"#tab-all-artifacts\">All artifacts</a></li>" + \
            "</ul>" + \
            "<div id=\"tab-definition\">" + \
            "<h2>Artifact sources</h2>").format(report_name=report_name, script="{ $(\"#tabs\").tabs(); }")

    examples = ""
    i = 1
    for artifact_source in config.artifactSources:
        html += "<div class=\"artifact-source\">"
        if artifact_source["type"] == "dependency-graph":
            html += "<h3>Dependency graph #%i</h3><h4>Roots</h4><ul>" % i
            i += 1
            for root in sorted(artifact_source['top-level-gavs']):
                ma = MavenArtifact.createFromGAV(root)
                gid = ma.groupId
                aid = ma.artifactId
                ver = ma.version
                if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                    if ma.is_example():
                        examples += "<li class=\"error\"><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
                                    gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
                    else:
                        html += "<li><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
                                gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
                else:
                    if ma.is_example():
                        examples += "<li class=\"example\">{gid}{colon}{aid}{colon}{ver}</li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
                    else:
                        html += "<li class=\"error\">{gid}{colon}{aid}{colon}{ver}</li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
            html += examples + "</ul><h4>BOMs</h4><ul>"
            if len(artifact_source['injected-boms']):
                for bom in artifact_source['injected-boms']:
                    ma = MavenArtifact.createFromGAV(bom)
                    gid = ma.groupId
                    aid = ma.artifactId
                    ver = ma.version
                    if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                        html += "<li><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
                                gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
                    else:
                        html += "<li><span class=\"error\">{gid}{colon}{aid}{colon}{ver}</span></li>".format(
                                gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
            else:
                html += "<li><em>none</em></li>"
            if len(artifact_source['excluded-subgraphs']):
                html += "</ul><h4>Excluded subgraphs</h4><ul>"
                for exclusion in artifact_source['excluded-subgraphs']:
                    ma = MavenArtifact.createFromGAV(exclusion)
                    gid = ma.groupId
                    aid = ma.artifactId
                    ver = ma.version
                    if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                        html += "<li><a class=\"error\" href=\"pages/artifact_version_{gid}${aid}${ver}.html\">" + \
                                "{gid}{colon}{aid}{colon}{ver}</a></li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
                    else:
                        html += "<li class=\"excluded\">{gid}{colon}{aid}{colon}{ver}</li>".format(
                                gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
            if artifact_source['preset'] in ["sob-build", "scope-with-embedded", "requires", "managed-sob-build"]:
                contents = "runtime dependencies"
            elif artifact_source['preset'] in ["sob", "build-env", "build-requires", "br", "managed-sob"]:
                contents = "build-time dependencies"
            else:
                contents = artifact_source['preset']
            html += "</ul><h4>Contents</h4><ul><li>%s</li></ul>" % contents
        else:
            html += artifact_source["type"]
        html += "</div>"
    html += "<div class=\"artifact-source\"><h3>Global</h3><h4>Excluded GA(TC)V patterns</h4><ul>"
    if len(config.excludedGAVs):
        for excluded_pattern in config.excludedGAVs:
            html += "<li class=\"excluded\">%s</li>" % excluded_pattern
    else:
        html += "<li><em>none</em></li>"
    html += "</ul><h4>Excluded repositories</h4><ul>"
    if len(config.excludedRepositories):
        for excluded_repo in config.excludedRepositories:
            html += "<li class=\"excluded\"><a href=\"%(url)s\">%(url)s</a></li>" % {"url": excluded_repo}
    else:
        html += "<li><em>none</em></li>"
    html += "</ul></div>"

    html += "</div>\n<div id=\"tab-multi-versioned-artifacts\"><h2>Multi-versioned artifacts</h2><ul>"
    for groupid in sorted(multiversion_gas.keys()):
        artifactids = multiversion_gas[groupid]
        for artifactid in sorted(artifactids.keys()):
            html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
                     "<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>" + \
                     "</li><ul>").format(artifactid=artifactid, groupid=groupid, colon=COLON_HTML)
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += "<li><a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>".format(
                        version=version, gav_filename=gav.replace(":", "$"))
            html += "</ul>"

    html += "</ul></div>\n<div id=\"tab-malformed-versions\"><h2>Malformed versions</h2><ul>"
    for groupid in sorted(malformed_versions.keys()):
        artifactids = malformed_versions[groupid]
        for artifactid in sorted(artifactids.keys()):
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
                         "<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>{colon}" + \
                         "<a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>").format(artifactid=artifactid, groupid=groupid,
                                                                                                             version=version,
                                                                                                             gav_filename=gav.replace(":", "$"),
                                                                                                             colon=COLON_HTML)

    html += "</ul></div>\n<div id=\"tab-optional-artifacts\"><h2>Optional artifacts</h2><h3>Direct optionals</h3><ul>"
    for ma in sorted(optional_artifacts.keys()):
        if optional_artifacts[ma]:
            gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
            html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\" class=\"optional\">{groupid}</a>{colon}" + \
                     "<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\" class=\"optional\">{artifactid}</a>{colon}" + \
                     "<a href=\"pages/artifact_version_{gav_filename}.html\" class=\"optional\">{version}</a></li>").format(groupid=ma.groupId,
                                                                                                         artifactid=ma.artifactId,
                                                                                                         version=ma.version,
                                                                                                         gav_filename=gav_filename,
                                                                                                         colon=COLON_HTML)
    html += "</ul><h3>Transitive optionals</h3><ul>"
    for ma in sorted(optional_artifacts.keys()):
        if not optional_artifacts[ma]:
            gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
            html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\" class=\"optional\">{groupid}</a>{colon}" + \
                     "<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\" class=\"optional\">{artifactid}</a>{colon}" + \
                     "<a href=\"pages/artifact_version_{gav_filename}.html\" class=\"optional\">{version}</a></li>").format(groupid=ma.groupId,
                                                                                                         artifactid=ma.artifactId,
                                                                                                         version=ma.version,
                                                                                                         gav_filename=gav_filename,
                                                                                                         colon=COLON_HTML)
    html += "</ul></div>\n<div id=\"tab-all-artifacts\"><h2>All artifacts</h2><ul>"
    for groupid in sorted(groupids.keys()):
        artifactids = groupids[groupid]
        for artifactid in sorted(artifactids.keys()):
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
                         "<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>{colon}" + \
                         "<a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>").format(groupid=groupid, artifactid=artifactid,
                                                                                                             version=version, gav_filename=gav.replace(":", "$"), colon=COLON_HTML)
    html += "</ul></div></div></div></body></html>"
    with open(os.path.join(output, "index.html"), "w") as htmlfile:
        htmlfile.write(html)
    def _listDependencies(self, repoUrls, gavs):
        """
        Loads maven artifacts from mvn dependency:list.

        :param repoUrls: URL of the repositories that contains the listed artifacts
        :param gavs: List of top level GAVs
        :returns: Dictionary where index is MavenArtifact object and value is
                  it's repo root URL, or empty dictionary if something goes wrong.
        """
        artifacts = {}

        for gav in gavs:
            logging.debug("Resolving dependencies for %s", gav)
            artifact = MavenArtifact.createFromGAV(gav)

            pomFilename = 'poms/' + artifact.getPomFilename()
            successPomUrl = None
            fetched = False
            for repoUrl in repoUrls:
                pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
                fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
                if fetched:
                    successPomUrl = repoUrl
                    break

            if not fetched:
                logging.warning("Failed to retrieve pom file for artifact %s", gav)
                continue

            tempDir = maven_repo_util.getTempDir()
            if not os.path.exists(tempDir):
                os.makedirs(tempDir)

            # Create settings.xml
            settingsFile = tempDir + "settings.xml"
            with open(settingsFile, 'w') as settings:
                settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL)
                settings.write(settingsContent)

            # Build dependency:list
            depsDir = tempDir + "maven-deps-output/"
            outFile = depsDir + gav + ".out"
            args = ['mvn', 'dependency:list', '-N',
                                              '-DoutputFile=' + outFile,
                                              '-f', pomFilename,
                                              '-s', settingsFile]
            logging.debug("Running Maven:\n  %s", " ".join(args))
            mvn = Popen(args, stdout=PIPE)
            mvnStdout = mvn.communicate()[0]
            logging.debug("Maven output:\n%s", mvnStdout)

            if mvn.returncode != 0:
                logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
                continue

            with open(outFile, 'r') as out:
                depLines = out.readlines()

            gavList = self._parseDepList(depLines)
            newArtifacts = self._listArtifacts(repoUrls, gavList)

            if self.configuration.allClassifiers:
                for artifact in newArtifacts.keys():
                    spec = newArtifacts[artifact]
                    out = self._lftpFind(spec.url + artifact.getDirPath())

                    files = []
                    for line in out.split('\n'):
                        if line != "./" and line != "":
                            files.append(line[2:])

                    (extsAndClass, suffix) = self._getExtensionsAndClassifiers(
                        artifact.artifactId, artifact.version, files)
                    if len(extsAndClass) > 1 and "pom" in extsAndClass:
                        del extsAndClass["pom"]
                    spec.classifiers = extsAndClass[artifact.artifactType]
                    del extsAndClass[artifact.artifactType]
                    self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId,
                                      artifact.version, extsAndClass, suffix, spec.url)

            artifacts.update(newArtifacts)

        return artifacts
    def _listDependencies(self, repoUrls, gavs, recursive, skipmissing):
        """
        Loads maven artifacts from mvn dependency:list.

        :param repoUrls: URL of the repositories that contains the listed artifacts
        :param gavs: List of top level GAVs
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        artifacts = {}
        workingSet = set(gavs)
        checkedSet = set()

        while workingSet:
            gav = workingSet.pop()
            checkedSet.add(gav)
            logging.debug("Resolving dependencies for %s", gav)
            artifact = MavenArtifact.createFromGAV(gav)

            pomFilename = 'poms/' + artifact.getPomFilename()
            successPomUrl = None
            fetched = False
            for repoUrl in repoUrls:
                pomUrl = maven_repo_util.slashAtTheEnd(
                    repoUrl) + artifact.getPomFilepath()
                fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
                if fetched:
                    successPomUrl = repoUrl
                    break

            if not fetched:
                logging.warning("Failed to retrieve pom file for artifact %s",
                                gav)
                continue

            tempDir = maven_repo_util.getTempDir()
            if not os.path.exists(tempDir):
                os.makedirs(tempDir)

            # Create settings.xml
            settingsFile = tempDir + "settings.xml"
            settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
                                               .replace('${temp}', maven_repo_util.getTempDir())
            with open(settingsFile, 'w') as settings:
                settings.write(settingsContent)

            # Build dependency:list
            depsDir = tempDir + "maven-deps-output/"
            outFile = depsDir + gav + ".out"
            args = [
                'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile,
                '-f', pomFilename, '-s', settingsFile
            ]
            logging.debug("Running Maven:\n  %s", " ".join(args))
            logging.debug("settings.xml contents: %s", settingsContent)
            mvn = Popen(args, stdout=PIPE)
            mvnStdout = mvn.communicate()[0]
            logging.debug("Maven output:\n%s", mvnStdout)

            if mvn.returncode != 0:
                logging.warning(
                    "Maven failed to finish with success. Skipping artifact %s",
                    gav)
                continue

            with open(outFile, 'r') as out:
                depLines = out.readlines()
            gavList = self._parseDepList(depLines)
            logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))

            newArtifacts = self._listArtifacts(repoUrls, gavList)

            if recursive:
                for artifact in newArtifacts:
                    ngav = artifact.getGAV()
                    if ngav not in checkedSet:
                        workingSet.add(ngav)

            if self.configuration.allClassifiers:
                for artifact in newArtifacts.keys():
                    spec = newArtifacts[artifact]
                    try:
                        out = self._lftpFind(spec.url + artifact.getDirPath())
                    except IOError as ex:
                        if skipmissing:
                            logging.warn(
                                "Error while listing files in %s: %s. Skipping...",
                                spec.url + artifact.getDirPath(), str(ex))
                            continue
                        else:
                            raise ex

                    files = []
                    for line in out.split('\n'):
                        if line != "./" and line != "":
                            files.append(line[2:])

                    (extsAndClass, suffix) = self._getExtensionsAndClassifiers(
                        artifact.artifactId, artifact.version, files)
                    if len(extsAndClass) > 1 and "pom" in extsAndClass:
                        del extsAndClass["pom"]
                    if artifact.artifactType in extsAndClass:
                        spec.classifiers = extsAndClass[artifact.artifactType]
                        del extsAndClass[artifact.artifactType]
                        self._addArtifact(newArtifacts, artifact.groupId,
                                          artifact.artifactId,
                                          artifact.version, extsAndClass,
                                          suffix, spec.url)
                    else:
                        if files:
                            logging.warn(
                                "Main artifact is missing in filelist listed from %s. Files were:\n%s",
                                spec.url + artifact.getDirPath(),
                                "\n".join(files))
                        else:
                            logging.warn(
                                "An empty filelist was listed from %s. Skipping...",
                                spec.url + artifact.getDirPath())

            artifacts.update(newArtifacts)

        return artifacts
示例#44
0
    def _listDependencyGraph(self, indyUrl, wsid, sourceKey, gavs, excludedSources=[], excludedSubgraphs=[],
                             preset="requires", mutator=None, patcherIds=[], injectedBOMs=[], analyze=False):
        """
        Loads maven artifacts from dependency graph.

        :param indyUrl: URL of the Indy instance
        :param wsid: workspace ID
        :param sourceKey: the Indy artifact source key, consisting of the source type and
                          its name of the form <{repository|deploy|group}:<name>>
        :param gavs: List of top level GAVs
        :param excludedSources: list of excluded sources' keys
        :param excludedSubgraphs: list of artifacts' GAVs which we want to exclude along with their subgraphs
        :param preset: preset used while creating the urlmap
        :param mutator: mutator used for dependency graph mutation
        :param patcherIds: list of patcher ID strings for Indy
        :param injectedBOMs: list of injected BOMs used with dependency management injection
                             Maven extension
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        indy = IndyApi(indyUrl)

        if not preset:
            preset = "requires"  # only runtime dependencies

        if analyze and not wsid:
            _wsid = "temp"
        else:
            _wsid = wsid

        # Resolve graph MANIFEST for GAVs
        if self.configuration.useCache:
            urlmap = indy.urlmap(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
                                 excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
        else:
            urlmap = indy.urlmap_nocache(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
                                         excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)

        # parse returned map
        artifacts = {}
        if "projects" in urlmap:
            urlmap = urlmap["projects"]
        for gav in urlmap:
            artifact = MavenArtifact.createFromGAV(gav)
            groupId = artifact.groupId
            artifactId = artifact.artifactId
            version = artifact.version

            filenames = urlmap[gav]["files"]
            url = urlmap[gav]["repoUrl"]

            (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)

            self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)

        if analyze:
            gas = []
            for ma in artifacts.keys():
                ga = ma.getGA()
                if not ga in gas:
                    gas.append(ga)
            try:
                if self.configuration.useCache:
                    path_dict = indy.paths(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs, preset,
                                           mutator, patcherIds, injectedBOMs, False)
                else:
                    path_dict = indy.paths_nocache(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs,
                                                   preset, mutator, patcherIds, injectedBOMs, False)
            except RuntimeError:
                path_dict = dict()
            if "projects" in path_dict:
                path_dict = path_dict["projects"]
            if path_dict:
                for ma in artifacts.keys():
                    for key in path_dict.keys():
                        key_gav = key
                        if key.count(":") > 2:
                            key_split = key.split(":")
                            key_gav = ":".join([key_split[0], key_split[1], key_split[-1]])
                        if ma.getGAV() == key_gav:
                            gav_paths = path_dict[key]
                            if type(gav_paths) is dict:
                                gav_paths = gav_paths["paths"]
                            for gav_path in gav_paths:
                                direct = True
                                if type(gav_path) is dict:
                                    gav_path = gav_path["pathParts"]
                                rel_path = []
                                for gav_rel in gav_path:
                                    if "inherited" in gav_rel and gav_rel["inherited"]:
                                        direct = False
                                        break
                                    declaring = MavenArtifact.createFromGAV(gav_rel["declaring"])
                                    target = MavenArtifact.createFromGAV(maven_repo_util.gatvc_to_gatcv(gav_rel["target"]))
                                    rel_type = gav_rel["type"] if "type" in gav_rel else gav_rel["rel"]
                                    if rel_type == "DEPENDENCY":
                                        extra = gav_rel["scope"]
                                        if "optional" in gav_rel and gav_rel["optional"]:
                                            extra = "%s %s" % (extra, "optional")
                                        rel = ArtifactRelationship(declaring, target, rel_type, extra)
                                    elif rel_type == "PLUGIN_DEP":
                                        rel = ArtifactRelationship(declaring, target, rel_type, gav_rel["plugin"])
                                    elif rel_type == "BOM":
                                        if "mixin" in gav_rel and gav_rel["mixin"]:
                                            direct = False
                                            break
                                        rel = ArtifactRelationship(declaring, target, rel_type)
                                    else:
                                        rel = ArtifactRelationship(declaring, target, rel_type)
                                    rel_path.append(rel)

                                if direct:
                                    artifacts[ma].add_path(rel_path)
            for ma in artifacts.keys():
                if not artifacts[ma].paths and ma.getGAV() not in gavs:
                    # create artificial unknown paths from all roots to current artifact
                    for root in gavs:
                        if root != ma.getGAV():
                            rel_path = [ArtifactRelationship(MavenArtifact.createFromGAV(root), None, None),
                                        ArtifactRelationship(None, ma, None)]
                            artifacts[ma].add_path(rel_path)

            if not wsid:
                try:
                    indy.deleteWorkspace(_wsid)
                except Exception as err:
                    logging.warning("Workspace deletion failed: %s" % str(err))

        return artifacts
    def _filterExcludedTypes(self, artifactList):
        '''
        Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
        not-excluded artifact types only.

        :param artifactList: artifactList to be filtered.
        :param exclTypes: list of excluded types
        :returns: artifactList without artifacts that matched specified types and had no other main types.
        '''
        logging.debug("Filtering artifacts with excluded types.")
        regExps = maven_repo_util.getRegExpsFromStrings(
            self.config.gatcvWhitelist)
        exclTypes = self.config.excludedTypes
        for ga in artifactList.keys():
            for priority in artifactList[ga].keys():
                for version in artifactList[ga][priority].keys():
                    artSpec = artifactList[ga][priority][version]
                    for artType in list(artSpec.artTypes.keys()):
                        if artType in exclTypes:
                            artTypeObj = artSpec.artTypes[artType]
                            classifiers = artTypeObj.classifiers
                            (groupId, artifactId) = ga.split(':')
                            for classifier in list(classifiers):
                                art = MavenArtifact(groupId, artifactId,
                                                    artType, version,
                                                    classifier)
                                gatcv = art.getGATCV()
                                if not maven_repo_util.somethingMatch(
                                        regExps, gatcv):
                                    logging.debug(
                                        "Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
                                        "excluded type.", classifier, ga,
                                        artType, version, priority)
                                    classifiers.remove(classifier)
                                else:
                                    logging.debug(
                                        "Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
                                        "whitelist pattern.", ga, artType,
                                        classifier, version, priority)
                            if not classifiers:
                                logging.debug(
                                    "Dropping %s:%s:%s from priority %i because of no classifier left.",
                                    ga, artType, version, priority)
                                del (artSpec.artTypes[artType])
                    noMain = True
                    for artType in artSpec.artTypes.keys():
                        artTypeObj = artSpec.artTypes[artType]
                        if artTypeObj.mainType:
                            noMain = False
                            break
                    if not artSpec.artTypes or noMain:
                        if noMain:
                            logging.debug(
                                "Dropping GAV %s:%s from priority %i because of no main artifact left.",
                                ga, version, priority)
                        else:
                            logging.debug(
                                "Dropping GAV %s:%s from priority %i because of no artifact type left.",
                                ga, version, priority)
                        del artifactList[ga][priority][version]
                if not artifactList[ga][priority]:
                    logging.debug(
                        "Dropping GA %s from priority %i because of no version left.",
                        ga, priority)
                    del artifactList[ga][priority]
            if not artifactList[ga]:
                logging.debug("Dropping GA %s because of no priority left.",
                              ga)
                del artifactList[ga]
        return artifactList
    def _listDependencyGraph(self, cartoUrl, wsid, sourceKey, gavs, excludedSources=[], excludedSubgraphs=[],
                             preset="requires", mutator=None, patcherIds=[], injectedBOMs=[], analyze=False):
        """
        Loads maven artifacts from dependency graph.

        :param cartoUrl: URL of the Cartographer instance
        :param wsid: workspace ID
        :param sourceKey: the Cartographer artifact source key, consisting of the source type and
                          its name of the form <{repository|deploy|group}:<name>>
        :param gavs: List of top level GAVs
        :param excludedSources: list of excluded sources' keys
        :param excludedSubgraphs: list of artifacts' GAVs which we want to exclude along with their subgraphs
        :param preset: preset used while creating the urlmap
        :param mutator: mutator used for dependency graph mutation
        :param patcherIds: list of patcher ID strings for Cartographer
        :param injectedBOMs: list of injected BOMs used with dependency management injection
                             Maven extension
        :returns: Dictionary where index is MavenArtifact object and value is
                  ArtifactSpec with its repo root URL
        """
        carto = CartoClient(cartoUrl)

        if not preset:
            preset = "requires"  # only runtime dependencies

        if analyze and not wsid:
            _wsid = "temp"
        else:
            _wsid = wsid

        # Resolve graph MANIFEST for GAVs
        if self.configuration.useCache:
            urlmap = carto.urlmap(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
                                  excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
        else:
            urlmap = carto.urlmap_nocache(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
                                          excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)

        # parse returned map
        artifacts = {}
        if "projects" in urlmap:
            urlmap = urlmap["projects"]
        for gav in urlmap:
            artifact = MavenArtifact.createFromGAV(gav)
            groupId = artifact.groupId
            artifactId = artifact.artifactId
            version = artifact.version

            filenames = urlmap[gav]["files"]
            url = urlmap[gav]["repoUrl"]

            (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)

            self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)

        if analyze:
            gas = []
            for ma in artifacts.keys():
                ga = ma.getGA()
                if not ga in gas:
                    gas.append(ga)
            try:
                if self.configuration.useCache:
                    path_dict = carto.paths(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs, preset,
                                            mutator, patcherIds, injectedBOMs, False)
                else:
                    path_dict = carto.paths_nocache(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs,
                                                    preset, mutator, patcherIds, injectedBOMs, False)
            except RuntimeError:
                path_dict = dict()
            if "projects" in path_dict:
                path_dict = path_dict["projects"]
            if path_dict:
                for ma in artifacts.keys():
                    for key in path_dict.keys():
                        key_gav = key
                        if key.count(":") > 2:
                            key_split = key.split(":")
                            key_gav = ":".join([key_split[0], key_split[1], key_split[-1]])
                        if ma.getGAV() == key_gav:
                            gav_paths = path_dict[key]
                            if type(gav_paths) is dict:
                                gav_paths = gav_paths["paths"]
                            for gav_path in gav_paths:
                                direct = True
                                if type(gav_path) is dict:
                                    gav_path = gav_path["pathParts"]
                                rel_path = []
                                for gav_rel in gav_path:
                                    if "inherited" in gav_rel and gav_rel["inherited"]:
                                        direct = False
                                        break
                                    declaring = MavenArtifact.createFromGAV(gav_rel["declaring"])
                                    target = MavenArtifact.createFromGAV(maven_repo_util.gatvc_to_gatcv(gav_rel["target"]))
                                    rel_type = gav_rel["type"] if "type" in gav_rel else gav_rel["rel"]
                                    if rel_type == "DEPENDENCY":
                                        extra = gav_rel["scope"]
                                        if "optional" in gav_rel and gav_rel["optional"]:
                                            extra = "%s %s" % (extra, "optional")
                                        rel = ArtifactRelationship(declaring, target, rel_type, extra)
                                    elif rel_type == "PLUGIN_DEP":
                                        rel = ArtifactRelationship(declaring, target, rel_type, gav_rel["plugin"])
                                    elif rel_type == "BOM":
                                        if "mixin" in gav_rel and gav_rel["mixin"]:
                                            direct = False
                                            break
                                        rel = ArtifactRelationship(declaring, target, rel_type)
                                    else:
                                        rel = ArtifactRelationship(declaring, target, rel_type)
                                    rel_path.append(rel)

                                if direct:
                                    artifacts[ma].add_path(rel_path)
            for ma in artifacts.keys():
                if not artifacts[ma].paths and ma.getGAV() not in gavs:
                    # create artificial unknown paths from all roots to current artifact
                    for root in gavs:
                        if root != ma.getGAV():
                            rel_path = [ArtifactRelationship(MavenArtifact.createFromGAV(root), None, None),
                                        ArtifactRelationship(None, ma, None)]
                            artifacts[ma].add_path(rel_path)

            if not wsid:
                try:
                    carto.deleteWorkspace(_wsid)
                except Exception as err:
                    logging.warning("Workspace deletion failed: %s" % str(err))

        return artifacts
def generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts):
    html = (
        "<html><head><title>Repository {report_name}</title>"
        + '<link rel="stylesheet" type="text/css" href="http://code.jquery.com/ui/1.11.4/themes/smoothness/jquery-ui.css">'
        + '<script src="http://code.jquery.com/jquery-1.10.2.js"></script>'
        + '<script src="http://code.jquery.com/ui/1.11.4/jquery-ui.js"></script>'
        + '<link rel="stylesheet" type="text/css" href="pages/style.css">'
        + "<script>$(function() {script});</script></head><body>"
        + '<div class="artifact"><h1>{report_name}</h1><div id="tabs">'
        + "<ul>"
        + '<li><a href="#tab-definition">Artifact sources</a></li>'
        + '<li><a href="#tab-multi-versioned-artifacts">Multi-versioned artifacts</a></li>'
        + '<li><a href="#tab-malformed-versions">Malformed versions</a></li>'
        + '<li><a href="#tab-optional-artifacts">Optional artifacts</a></li>'
        + '<li><a href="#tab-all-artifacts">All artifacts</a></li>'
        + "</ul>"
        + '<div id="tab-definition">'
        + "<h2>Artifact sources</h2>"
    ).format(report_name=report_name, script='{ $("#tabs").tabs(); }')

    examples = ""
    i = 1
    for artifact_source in config.artifactSources:
        html += '<div class="artifact-source">'
        if artifact_source["type"] == "dependency-graph":
            html += "<h3>Dependency graph #%i</h3><h4>Roots</h4><ul>" % i
            i += 1
            for root in sorted(artifact_source["top-level-gavs"]):
                ma = MavenArtifact.createFromGAV(root)
                gid = ma.groupId
                aid = ma.artifactId
                ver = ma.version
                if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                    if ma.is_example():
                        examples += '<li class="error"><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
                    else:
                        html += '<li><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
                else:
                    if ma.is_example():
                        examples += '<li class="example">{gid}{colon}{aid}{colon}{ver}</li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
                    else:
                        html += '<li class="error">{gid}{colon}{aid}{colon}{ver}</li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
            html += examples + "</ul><h4>BOMs</h4><ul>"
            if len(artifact_source["injected-boms"]):
                for bom in artifact_source["injected-boms"]:
                    ma = MavenArtifact.createFromGAV(bom)
                    gid = ma.groupId
                    aid = ma.artifactId
                    ver = ma.version
                    if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                        html += '<li><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
                    else:
                        html += '<li><span class="error">{gid}{colon}{aid}{colon}{ver}</span></li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
            else:
                html += "<li><em>none</em></li>"
            if len(artifact_source["excluded-subgraphs"]):
                html += "</ul><h4>Excluded subgraphs</h4><ul>"
                for exclusion in artifact_source["excluded-subgraphs"]:
                    ma = MavenArtifact.createFromGAV(exclusion)
                    gid = ma.groupId
                    aid = ma.artifactId
                    ver = ma.version
                    if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
                        html += (
                            '<li><a class="error" href="pages/artifact_version_{gid}${aid}${ver}.html">'
                            + "{gid}{colon}{aid}{colon}{ver}</a></li>".format(
                                gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                            )
                        )
                    else:
                        html += '<li class="excluded">{gid}{colon}{aid}{colon}{ver}</li>'.format(
                            gid=gid, aid=aid, ver=ver, colon=COLON_HTML
                        )
            if artifact_source["preset"] in ["sob-build", "scope-with-embedded", "requires", "managed-sob-build"]:
                contents = "runtime dependencies"
            elif artifact_source["preset"] in ["sob", "build-env", "build-requires", "br", "managed-sob"]:
                contents = "build-time dependencies"
            else:
                contents = artifact_source["preset"]
            html += "</ul><h4>Contents</h4><ul><li>%s</li></ul>" % contents
        else:
            html += artifact_source["type"]
        html += "</div>"
    html += '<div class="artifact-source"><h3>Global</h3><h4>Excluded GA(TC)V patterns</h4><ul>'
    if len(config.excludedGAVs):
        for excluded_pattern in config.excludedGAVs:
            html += '<li class="excluded">%s</li>' % excluded_pattern
    else:
        html += "<li><em>none</em></li>"
    html += "</ul><h4>Excluded repositories</h4><ul>"
    if len(config.excludedRepositories):
        for excluded_repo in config.excludedRepositories:
            html += '<li class="excluded"><a href="%(url)s">%(url)s</a></li>' % {"url": excluded_repo}
    else:
        html += "<li><em>none</em></li>"
    html += "</ul></div>"

    html += '</div>\n<div id="tab-multi-versioned-artifacts"><h2>Multi-versioned artifacts</h2><ul>'
    for groupid in sorted(multiversion_gas.keys()):
        artifactids = multiversion_gas[groupid]
        for artifactid in sorted(artifactids.keys()):
            html += (
                '<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
                + '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>'
                + "</li><ul>"
            ).format(artifactid=artifactid, groupid=groupid, colon=COLON_HTML)
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += '<li><a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'.format(
                    version=version, gav_filename=gav.replace(":", "$")
                )
            html += "</ul>"

    html += '</ul></div>\n<div id="tab-malformed-versions"><h2>Malformed versions</h2><ul>'
    for groupid in sorted(malformed_versions.keys()):
        artifactids = malformed_versions[groupid]
        for artifactid in sorted(artifactids.keys()):
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += (
                    '<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
                    + '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>{colon}'
                    + '<a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'
                ).format(
                    artifactid=artifactid,
                    groupid=groupid,
                    version=version,
                    gav_filename=gav.replace(":", "$"),
                    colon=COLON_HTML,
                )

    html += '</ul></div>\n<div id="tab-optional-artifacts"><h2>Optional artifacts</h2><h3>Direct optionals</h3><ul>'
    for ma in sorted(optional_artifacts.keys()):
        if optional_artifacts[ma]:
            gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
            html += (
                '<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}" class="optional">{groupid}</a>{colon}'
                + '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}" class="optional">{artifactid}</a>{colon}'
                + '<a href="pages/artifact_version_{gav_filename}.html" class="optional">{version}</a></li>'
            ).format(
                groupid=ma.groupId,
                artifactid=ma.artifactId,
                version=ma.version,
                gav_filename=gav_filename,
                colon=COLON_HTML,
            )
    html += "</ul><h3>Transitive optionals</h3><ul>"
    for ma in sorted(optional_artifacts.keys()):
        if not optional_artifacts[ma]:
            gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
            html += (
                '<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}" class="optional">{groupid}</a>{colon}'
                + '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}" class="optional">{artifactid}</a>{colon}'
                + '<a href="pages/artifact_version_{gav_filename}.html" class="optional">{version}</a></li>'
            ).format(
                groupid=ma.groupId,
                artifactid=ma.artifactId,
                version=ma.version,
                gav_filename=gav_filename,
                colon=COLON_HTML,
            )
    html += '</ul></div>\n<div id="tab-all-artifacts"><h2>All artifacts</h2><ul>'
    for groupid in sorted(groupids.keys()):
        artifactids = groupids[groupid]
        for artifactid in sorted(artifactids.keys()):
            artifacts = artifactids[artifactid]
            for version in sorted(artifacts.keys()):
                gav = "%s:%s:%s" % (groupid, artifactid, version)
                html += (
                    '<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
                    + '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>{colon}'
                    + '<a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'
                ).format(
                    groupid=groupid,
                    artifactid=artifactid,
                    version=version,
                    gav_filename=gav.replace(":", "$"),
                    colon=COLON_HTML,
                )
    html += "</ul></div></div></div></body></html>"
    with open(os.path.join(output, "index.html"), "w") as htmlfile:
        htmlfile.write(html)