def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
pomMain = True
# The pom is main only if no other main artifact is available
if len(extsAndClass) > 1 and self._containsMainArtifact(extsAndClass) and "pom" in extsAndClass:
pomMain = False
artTypes = []
for ext, classifiers in extsAndClass.iteritems():
main = ext == "pom" and pomMain
if not main:
for classifier in classifiers:
extClassifier = "%s:%s" % (ext, classifier or "")
main = extClassifier not in self.notMainExtClassifiers
if main:
break
artTypes.append(ArtifactType(ext, main, classifiers))
mavenArtifact = MavenArtifact(groupId, artifactId, None, version)
if suffix is not None:
mavenArtifact.snapshotVersionSuffix = suffix
if mavenArtifact in artifacts:
artifacts[mavenArtifact].merge(ArtifactSpec(url, artTypes))
else:
logging.debug("Adding artifact %s", str(mavenArtifact))
artifacts[mavenArtifact] = ArtifactSpec(url, artTypes)
def test_maven_artifact(self):
artifact1 = MavenArtifact.createFromGAV("org.jboss:jboss-parent:pom:10")
self.assertEqual(artifact1.groupId, "org.jboss")
self.assertEqual(artifact1.artifactId, "jboss-parent")
self.assertEqual(artifact1.version, "10")
self.assertEqual(artifact1.getArtifactType(), "pom")
self.assertEqual(artifact1.getClassifier(), "")
self.assertEqual(artifact1.getArtifactFilename(), "jboss-parent-10.pom")
self.assertEqual(artifact1.getArtifactFilepath(), "org/jboss/jboss-parent/10/jboss-parent-10.pom")
artifact2 = MavenArtifact.createFromGAV("org.jboss:jboss-foo:jar:1.0")
self.assertEqual(artifact2.getArtifactFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.jar")
self.assertEqual(artifact2.getPomFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0.pom")
self.assertEqual(artifact2.getSourcesFilepath(), "org/jboss/jboss-foo/1.0/jboss-foo-1.0-sources.jar")
artifact3 = MavenArtifact.createFromGAV("org.jboss:jboss-test:jar:client:2.0.0.Beta1")
self.assertEqual(artifact3.getClassifier(), "client")
self.assertEqual(artifact3.getArtifactFilename(), "jboss-test-2.0.0.Beta1-client.jar")
self.assertEqual(artifact3.getArtifactFilepath(),
"org/jboss/jboss-test/2.0.0.Beta1/jboss-test-2.0.0.Beta1-client.jar")
artifact4 = MavenArtifact.createFromGAV("org.acme:jboss-bar:jar:1.0-alpha-1:compile")
self.assertEqual(artifact4.getArtifactFilepath(), "org/acme/jboss-bar/1.0-alpha-1/jboss-bar-1.0-alpha-1.jar")
artifact5 = MavenArtifact.createFromGAV("com.google.guava:guava:pom:r05")
self.assertEqual(artifact5.groupId, "com.google.guava")
self.assertEqual(artifact5.artifactId, "guava")
self.assertEqual(artifact5.version, "r05")
self.assertEqual(artifact5.getArtifactType(), "pom")
self.assertEqual(artifact5.getClassifier(), "")
self.assertEqual(artifact5.getArtifactFilename(), "guava-r05.pom")
def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
pomMain = True
# The pom is main only if no other main artifact is available
if len(extsAndClass) > 1 and self._containsMainArtifact(extsAndClass) and "pom" in extsAndClass:
pomMain = False
artTypes = []
for ext, classifiers in extsAndClass.iteritems():
main = ext == "pom" and pomMain
if not main:
for classifier in classifiers:
extClassifier = "%s:%s" % (ext, classifier or "")
main = extClassifier not in self.notMainExtClassifiers
if main:
break
artTypes.append(ArtifactType(ext, main, classifiers))
mavenArtifact = MavenArtifact(groupId, artifactId, None, version)
if suffix is not None:
mavenArtifact.snapshotVersionSuffix = suffix
if mavenArtifact in artifacts:
artifacts[mavenArtifact].merge(ArtifactSpec(url, artTypes))
else:
logging.debug("Adding artifact %s", str(mavenArtifact))
artifacts[mavenArtifact] = ArtifactSpec(url, artTypes)
def _addArtifact(self, artifacts, groupId, artifactId, version, extsAndClass, suffix, url):
if len(extsAndClass) > 1 and self._containsNonPomWithoutClassifier(extsAndClass) and "pom" in extsAndClass:
del extsAndClass["pom"]
for ext in extsAndClass:
mavenArtifact = MavenArtifact(groupId, artifactId, ext, version)
if suffix is not None:
mavenArtifact.snapshotVersionSuffix = suffix
logging.debug("Adding artifact %s", str(mavenArtifact))
artifacts[mavenArtifact] = ArtifactSpec(url, extsAndClass[ext])
def _filterExcludedTypes(self, artifactList):
'''
Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
not-excluded artifact types only.
:param artifactList: artifactList to be filtered.
:param exclTypes: list of excluded types
:returns: artifactList without artifacts that matched specified types and had no other main types.
'''
logging.debug("Filtering artifacts with excluded types.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.gatcvWhitelist)
exclTypes = self.config.excludedTypes
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artSpec = artifactList[ga][priority][version]
for artType in list(artSpec.artTypes.keys()):
if artType in exclTypes:
artTypeObj = artSpec.artTypes[artType]
classifiers = artTypeObj.classifiers
(groupId, artifactId) = ga.split(':')
for classifier in list(classifiers):
art = MavenArtifact(groupId, artifactId, artType, version, classifier)
gatcv = art.getGATCV()
if not maven_repo_util.somethingMatch(regExps, gatcv):
logging.debug("Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
"excluded type.", classifier, ga, artType, version, priority)
classifiers.remove(classifier)
else:
logging.debug("Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
"whitelist pattern.", ga, artType, classifier, version, priority)
if not classifiers:
logging.debug("Dropping %s:%s:%s from priority %i because of no classifier left.", ga,
artType, version, priority)
del(artSpec.artTypes[artType])
noMain = True
for artType in artSpec.artTypes.keys():
artTypeObj = artSpec.artTypes[artType]
if artTypeObj.mainType:
noMain = False
break
if not artSpec.artTypes or noMain:
if noMain:
logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
else:
logging.debug("Dropping GAV %s:%s from priority %i because of no artifact type left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.", ga)
del artifactList[ga]
return artifactList
def _addArtifact(self, artifacts, groupId, artifactId, version,
extsAndClass, suffix, url):
if len(extsAndClass) > 1 and self._containsNonPomWithoutClassifier(
extsAndClass) and "pom" in extsAndClass:
del extsAndClass["pom"]
for ext in extsAndClass:
mavenArtifact = MavenArtifact(groupId, artifactId, ext, version)
if suffix is not None:
mavenArtifact.snapshotVersionSuffix = suffix
logging.debug("Adding artifact %s", str(mavenArtifact))
artifacts[mavenArtifact] = ArtifactSpec(url, extsAndClass[ext])
def test_listDependencies(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['http://repo.maven.apache.org/maven2/']
gavs = [
'com.sun.faces:jsf-api:2.0.11',
'org.apache.ant:ant:1.8.0'
]
dependencies = {
'javax.servlet:javax.servlet-api:jar:3.0.1': set(['javadoc', 'sources']),
'javax.servlet.jsp.jstl:jstl-api:jar:1.2': set(['javadoc', 'sources']),
'xml-apis:xml-apis:jar:1.3.04': set(['source', 'sources']),
'javax.servlet:servlet-api:jar:2.5': set(['sources']),
'javax.el:javax.el-api:jar:2.2.1': set(['javadoc', 'sources']),
'junit:junit:jar:3.8.2': set(['javadoc', 'sources']),
'xerces:xercesImpl:jar:2.9.0': set([]),
'javax.servlet.jsp:jsp-api:jar:2.1': set(['sources']),
'javax.servlet.jsp:javax.servlet.jsp-api:jar:2.2.1': set(['javadoc', 'sources']),
'org.apache.ant:ant-launcher:jar:1.8.0': set([])
}
expectedArtifacts = {}
for dep in dependencies:
artifact = MavenArtifact.createFromGAV(dep)
expectedArtifacts[artifact] = ArtifactSpec(repoUrls[0], dependencies[dep])
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listDependencies(repoUrls, gavs, False, False)
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def generateArtifactList(options):
"""
Generates artifact "list" from sources defined in the given configuration in options. The result
is dictionary with following structure:
<repo url> (string)
L artifacts (list of MavenArtifact)
"""
options.allclassifiers = (options.classifiers == '__all__')
artifactList = _generateArtifactList(options)
#build sane structure - url to MavenArtifact list
urlToMAList = {}
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
url = artSpec.url
for classifier in artSpec.classifiers:
if classifier == "" or options.allclassifiers:
artifact = MavenArtifact.createFromGAV(gat + (
(":" + classifier) if classifier else "") + ":" +
version)
urlToMAList.setdefault(url, []).append(artifact)
return urlToMAList
def generateArtifactList(options, args):
"""
Generates artifact "list" from sources defined in the given configuration in options. The result
is dictionary with following structure:
<repo url> (string)
L list of MavenArtifact
"""
artifactList = _generateArtifactList(options, args)
#build sane structure - url to MavenArtifact list
urlToMAList = {}
for ga in artifactList:
priorityList = artifactList[ga]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
url = artSpec.url
for artType in artSpec.artTypes.keys():
for classifier in artSpec.artTypes[artType].classifiers:
if classifier:
gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, version)
else:
gatcv = "%s:%s:%s" % (ga, artType, version)
artifact = MavenArtifact.createFromGAV(gatcv)
urlToMAList.setdefault(url, []).append(artifact)
return urlToMAList
def generateArtifactList(options):
"""
Generates artifact "list" from sources defined in the given configuration in options. The result
is dictionary with following structure:
<repo url> (string)
L artifacts (list of MavenArtifact)
"""
options.allclassifiers = (options.classifiers == '__all__')
artifactList = _generateArtifactList(options)
#build sane structure - url to MavenArtifact list
urlToMAList = {}
for gat in artifactList:
priorityList = artifactList[gat]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
url = artSpec.url
for classifier in artSpec.classifiers:
if classifier == "" or options.allclassifiers:
artifact = MavenArtifact.createFromGAV(gat + ((":" + classifier) if classifier else "") +
":" + version)
urlToMAList.setdefault(url, []).append(artifact)
return urlToMAList
def generateArtifactList(options, args):
"""
Generates artifact "list" from sources defined in the given configuration in options. The result
is dictionary with following structure:
<repo url> (string)
L list of MavenArtifact
"""
artifactList = _generateArtifactList(options, args)
#build sane structure - url to MavenArtifact list
urlToMAList = {}
for ga in artifactList:
priorityList = artifactList[ga]
for priority in priorityList:
versionList = priorityList[priority]
for version in versionList:
artSpec = versionList[version]
url = artSpec.url
for artType in artSpec.artTypes.keys():
for classifier in artSpec.artTypes[artType].classifiers:
if classifier:
gatcv = "%s:%s:%s:%s" % (ga, artType, classifier,
version)
else:
gatcv = "%s:%s:%s" % (ga, artType, version)
artifact = MavenArtifact.createFromGAV(gatcv)
urlToMAList.setdefault(url, []).append(artifact)
return urlToMAList
def test_listRepository_file(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['file://./tests/testrepo']
gavPatterns = ['bar:foo-bar:1.1', 'foo.baz:baz-core:1.0']
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]):
ArtifactSpec(repoUrls[0], set([])),
MavenArtifact.createFromGAV(gavPatterns[1]):
ArtifactSpec(repoUrls[0], set(['javadoc', 'sources']))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def _filterExcludedRepositories(self, artifactList, threadnum):
"""
Filter artifactList removing artifacts existing in specified repositories.
:param artifactList: artifactList to be filtered.
:returns: artifactList without artifacts that exists in specified repositories.
"""
logging.debug(
"Filtering artifacts contained in excluded repositories.")
pool = ThreadPool(threadnum)
# Contains artifact to be removed
delArtifacts = []
for ga in artifactList.keys():
groupId = ga.split(':')[0]
artifactId = ga.split(':')[1]
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artifact = MavenArtifact(groupId, artifactId, "pom",
version)
pool.apply_async(_artifactInRepos, [
self.config.excludedRepositories, artifact, priority,
delArtifacts
])
# Close the pool and wait for the workers to finnish
pool.close()
pool.join()
for artifact, priority in delArtifacts:
ga = artifact.getGA()
logging.debug(
"Dropping GAV %s:%s from priority %i because it was found in an excluded repository.",
ga, artifact.version, priority)
del artifactList[ga][priority][artifact.version]
if not artifactList[ga][priority]:
logging.debug(
"Dropping GA %s from priority %i because of no version left.",
ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.",
ga)
del artifactList[ga]
return artifactList
def test_listRepository_http(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['http://repo.maven.apache.org/maven2/']
gavPatterns = [
'com.sun.faces:jsf-api:2.0.11',
'org.apache.ant:ant:1.8.0'
]
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources'])),
MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set([]))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def test_listRepository_file(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['file://./tests/testrepo']
gavPatterns = [
'bar:foo-bar:1.1',
'foo.baz:baz-core:1.0'
]
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(repoUrls[0], set([])),
MavenArtifact.createFromGAV(gavPatterns[1]): ArtifactSpec(repoUrls[0], set(['javadoc', 'sources']))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def test_maven_artifact(self):
artifact1 = MavenArtifact.createFromGAV(
"org.jboss:jboss-parent:pom:10")
self.assertEqual(artifact1.groupId, "org.jboss")
self.assertEqual(artifact1.artifactId, "jboss-parent")
self.assertEqual(artifact1.version, "10")
self.assertEqual(artifact1.getArtifactType(), "pom")
self.assertEqual(artifact1.getClassifier(), "")
self.assertEqual(artifact1.getArtifactFilename(),
"jboss-parent-10.pom")
self.assertEqual(artifact1.getArtifactFilepath(),
"org/jboss/jboss-parent/10/jboss-parent-10.pom")
artifact2 = MavenArtifact.createFromGAV("org.jboss:jboss-foo:jar:1.0")
self.assertEqual(artifact2.getArtifactFilepath(),
"org/jboss/jboss-foo/1.0/jboss-foo-1.0.jar")
self.assertEqual(artifact2.getPomFilepath(),
"org/jboss/jboss-foo/1.0/jboss-foo-1.0.pom")
self.assertEqual(artifact2.getSourcesFilepath(),
"org/jboss/jboss-foo/1.0/jboss-foo-1.0-sources.jar")
artifact3 = MavenArtifact.createFromGAV(
"org.jboss:jboss-test:jar:client:2.0.0.Beta1")
self.assertEqual(artifact3.getClassifier(), "client")
self.assertEqual(artifact3.getArtifactFilename(),
"jboss-test-2.0.0.Beta1-client.jar")
self.assertEqual(
artifact3.getArtifactFilepath(),
"org/jboss/jboss-test/2.0.0.Beta1/jboss-test-2.0.0.Beta1-client.jar"
)
artifact4 = MavenArtifact.createFromGAV(
"org.acme:jboss-bar:jar:1.0-alpha-1:compile")
self.assertEqual(
artifact4.getArtifactFilepath(),
"org/acme/jboss-bar/1.0-alpha-1/jboss-bar-1.0-alpha-1.jar")
artifact5 = MavenArtifact.createFromGAV(
"com.google.guava:guava:pom:r05")
self.assertEqual(artifact5.groupId, "com.google.guava")
self.assertEqual(artifact5.artifactId, "guava")
self.assertEqual(artifact5.version, "r05")
self.assertEqual(artifact5.getArtifactType(), "pom")
self.assertEqual(artifact5.getClassifier(), "")
self.assertEqual(artifact5.getArtifactFilename(), "guava-r05.pom")
def findArtifact(gav, urls, artifacts):
artifact = MavenArtifact.createFromGAV(gav)
for url in urls:
if maven_repo_util.gavExists(url, artifact):
#Critical section?
artifacts[artifact] = ArtifactSpec(url, [ArtifactType(artifact.artifactType, True, set(['']))])
return
logging.warning('Artifact %s not found in any url!', artifact)
def test_listRepository_file_gatcvs(self):
config = configuration.Configuration()
config.addClassifiers = "__all__"
repoUrls = ['file://./tests/testrepo']
gatcvs = [
'bar:foo-bar:pom:1.1',
'foo.baz:baz-core:jar:1.0'
]
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, None, gatcvs)
expectedArtifacts = {
MavenArtifact.createFromGAV(gatcvs[0]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set(['']))]),
MavenArtifact.createFromGAV(gatcvs[1]): ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set([''])),
ArtifactType("jar", True, set(['', 'javadoc', 'sources']))])
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def test_listRepository_http(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['http://repo.maven.apache.org/maven2/']
gavPatterns = [
'com.sun.faces:jsf-api:2.0.11', 'org.apache.ant:ant:1.8.0'
]
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, gavPatterns)
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]):
ArtifactSpec(repoUrls[0], set(['javadoc', 'sources'])),
MavenArtifact.createFromGAV(gavPatterns[1]):
ArtifactSpec(repoUrls[0], set([]))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def findArtifact(gav, urls, artifacts):
artifact = MavenArtifact.createFromGAV(gav)
for url in urls:
if maven_repo_util.gavExists(url, artifact):
#Critical section?
artifacts[artifact] = ArtifactSpec(url)
return
logging.warning('Artifact %s not found in any url!', artifact)
def test_listRepository_file_gatcvs(self):
config = configuration.Configuration()
config.addClassifiers = "__all__"
repoUrls = ['file://./tests/testrepo']
gatcvs = ['bar:foo-bar:pom:1.1', 'foo.baz:baz-core:jar:1.0']
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listRepository(repoUrls, None, gatcvs)
expectedArtifacts = {
MavenArtifact.createFromGAV(gatcvs[0]):
ArtifactSpec(repoUrls[0], [ArtifactType("pom", True, set(['']))]),
MavenArtifact.createFromGAV(gatcvs[1]):
ArtifactSpec(repoUrls[0], [
ArtifactType("pom", True, set([''])),
ArtifactType("jar", True, set(['', 'javadoc', 'sources']))
])
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def _getExpectedArtifacts(self, repoUrl, dependencies):
artSpecDict = {}
for dep in dependencies.keys():
artifact = MavenArtifact.createFromGAV(dep)
artTypes = []
artTypes.append(ArtifactType(artifact.artifactType, artifact.artifactType != "pom", dependencies[dep]))
artSpec = ArtifactSpec(repoUrl, artTypes)
gav = artifact.getGAV()
if gav in artSpecDict.keys():
artSpecDict[gav].merge(artSpec)
else:
artSpecDict[gav] = artSpec
expectedArtifacts = {}
for (gav, artSpec) in artSpecDict.iteritems():
artifact = MavenArtifact.createFromGAV(gav)
expectedArtifacts[artifact] = artSpec
return expectedArtifacts
def depListToArtifactList(depList):
"""Convert the maven GAV to a URL relative path"""
regexComment = re.compile('#.*$')
#regexLog = re.compile('^\[\w*\]')
artifactList = []
for nextLine in depList:
nextLine = regexComment.sub('', nextLine)
nextLine = nextLine.strip()
gav = maven_repo_util.parseGATCVS(nextLine)
if gav:
artifactList.append(MavenArtifact.createFromGAV(gav))
return artifactList
def _filterExcludedRepositories(self, artifactList):
"""
Filter artifactList removing artifacts existing in specified repositories.
:param artifactList: artifactList to be filtered.
:returns: artifactList without artifacts that exists in specified repositories.
"""
logging.debug("Filtering artifacts contained in excluded repositories.")
pool = ThreadPool(maven_repo_util.MAX_THREADS)
# Contains artifact to be removed
delArtifacts = []
for ga in artifactList.keys():
groupId = ga.split(':')[0]
artifactId = ga.split(':')[1]
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artifact = MavenArtifact(groupId, artifactId, "pom", version)
pool.apply_async(
_artifactInRepos,
[self.config.excludedRepositories, artifact, priority, delArtifacts]
)
# Close the pool and wait for the workers to finnish
pool.close()
pool.join()
for artifact, priority in delArtifacts:
ga = artifact.getGA()
logging.debug("Dropping GAV %s:%s from priority %i because it was found in an excluded repository.",
ga, artifact.version, priority)
del artifactList[ga][priority][artifact.version]
if not artifactList[ga][priority]:
logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.", ga)
del artifactList[ga]
return artifactList
def test_listDependencies_recursive(self):
config = configuration.Configuration()
config.allClassifiers = True
repoUrls = ['http://repo.maven.apache.org/maven2/']
gavs = ['com.sun.faces:jsf-api:2.0.11', 'org.apache.ant:ant:1.8.0']
dependencies = {
'junit:junit:jar:3.8.2':
set(['sources', 'javadoc']),
'junit:junit:pom:3.8.2':
set(['sources', 'javadoc']),
'xerces:xercesImpl:jar:2.9.0':
set([]),
'xml-apis:xml-apis:jar:1.3.04':
set(['source', 'sources']),
'xml-apis:xml-apis:pom:1.3.04':
set(['source', 'sources']),
'javax.el:javax.el-api:jar:2.2.1':
set(['sources', 'javadoc']),
'javax.el:javax.el-api:pom:2.2.1':
set(['sources', 'javadoc']),
'xml-resolver:xml-resolver:jar:1.2':
set(['sources']),
'javax.servlet:servlet-api:jar:2.5':
set(['sources']),
'javax.servlet:servlet-api:pom:2.5':
set(['sources']),
'javax.servlet.jsp:jsp-api:jar:2.1':
set(['sources']),
'javax.servlet.jsp:jsp-api:pom:2.1':
set(['sources']),
'org.apache.ant:ant-launcher:jar:1.8.0':
set([]),
'javax.servlet.jsp.jstl:jstl-api:jar:1.2':
set(['sources', 'javadoc']),
'javax.servlet:javax.servlet-api:jar:3.0.1':
set(['sources', 'javadoc']),
'javax.servlet:javax.servlet-api:pom:3.0.1':
set(['sources', 'javadoc']),
'javax.servlet.jsp:javax.servlet.jsp-api:jar:2.2.1':
set(['sources', 'javadoc'])
}
expectedArtifacts = {}
for dep in dependencies:
artifact = MavenArtifact.createFromGAV(dep)
expectedArtifacts[artifact] = ArtifactSpec(repoUrls[0],
dependencies[dep])
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listDependencies(repoUrls, gavs, True,
False)
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def _getExpectedArtifacts(self, repoUrl, dependencies):
artSpecDict = {}
for dep in dependencies.keys():
artifact = MavenArtifact.createFromGAV(dep)
artTypes = []
artTypes.append(
ArtifactType(artifact.artifactType,
artifact.artifactType != "pom",
dependencies[dep]))
artSpec = ArtifactSpec(repoUrl, artTypes)
gav = artifact.getGAV()
if gav in artSpecDict.keys():
artSpecDict[gav].merge(artSpec)
else:
artSpecDict[gav] = artSpec
expectedArtifacts = {}
for (gav, artSpec) in artSpecDict.iteritems():
artifact = MavenArtifact.createFromGAV(gav)
expectedArtifacts[artifact] = artSpec
return expectedArtifacts
def depListToArtifactList(depList):
"""Convert the maven GAV to a URL relative path"""
regexComment = re.compile('#.*$')
#regexLog = re.compile('^\[\w*\]')
regexGAV = re.compile('(([\w\-.]+:){3}[\w\-.]+)(:[\w]*\S)?')
artifactList = []
for nextLine in depList:
nextLine = regexComment.sub('', nextLine)
nextLine = nextLine.strip()
gav = regexGAV.search(nextLine)
if gav:
artifactList.append(MavenArtifact(gav.group(0)))
return artifactList
def generate_report(output, config, artifact_list, report_name):
"""
Generates report. The report consists of a summary page, groupId pages, artifactId pages and leaf artifact pages.
Summary page contains list of roots, list of BOMs, list of multi-version artifacts, links to all artifacts and list
of artifacts that do not match the BOM version. Each artifact has a separate page containing paths from roots to
the artifact with path explanation.
"""
multiversion_gas = dict()
malformed_versions = dict()
if os.path.exists(output):
logging.warn("Target report path %s exists. Deleting...", output)
shutil.rmtree(output)
os.makedirs(os.path.join(output, "pages"))
roots = []
for artifact_source in config.artifactSources:
if artifact_source["type"] == "dependency-graph":
roots.extend(artifact_source["top-level-gavs"])
groupids = dict()
version_pattern = re.compile("^.*[.-]redhat-[^.]+$")
for ga in artifact_list:
(groupid, artifactid) = ga.split(":")
priority_list = artifact_list[ga]
for priority in priority_list:
versions = priority_list[priority]
if versions:
groupids.setdefault(groupid, dict()).setdefault(artifactid, dict()).update(versions)
if len(groupids[groupid][artifactid]) > 1:
multiversion_gas.setdefault(groupid, dict())[artifactid] = groupids[groupid][artifactid]
for version in versions:
if not version_pattern.match(version):
malformed_versions.setdefault(groupid, dict()).setdefault(artifactid, dict())[
version
] = groupids[groupid][artifactid][version]
optional_artifacts = dict()
for groupid in groupids.keys():
artifactids = groupids[groupid]
for artifactid in artifactids.keys():
versions = artifactids[artifactid]
for version in versions.keys():
art_spec = versions[version]
ma = MavenArtifact.createFromGAV("%s:%s:%s" % (groupid, artifactid, version))
generate_artifact_page(ma, roots, art_spec.paths, art_spec.url, output, groupids, optional_artifacts)
generate_artifactid_page(groupid, artifactid, versions, output)
generate_groupid_page(groupid, artifactids, output)
generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts)
generate_css(output)
def depListToArtifactList(depList):
"""Convert the maven GAV to a URL relative path"""
regexComment = re.compile('#.*$')
#regexLog = re.compile('^\[\w*\]')
# Match pattern groupId:artifactId:type:[classifier:]version[:scope]
regexGAV = re.compile('(([\w\-.]+:){3}([\w\-.]+:)?([\d][\w\-.]+))(:[\w]*\S)?')
artifactList = []
for nextLine in depList:
nextLine = regexComment.sub('', nextLine)
nextLine = nextLine.strip()
gav = regexGAV.search(nextLine)
if gav:
artifactList.append(MavenArtifact.createFromGAV(gav.group(1)))
return artifactList
def _filterExcludedRepositories(self, artifactList):
"""
Filter artifactList removing artifacts existing in specified repositories.
:param artifactList: artifactList to be filtered.
:returns: artifactList without arifacts that exists in specified repositories.
"""
logging.debug("Filtering artifacts contained in excluded repositories.")
pool = ThreadPool(maven_repo_util.MAX_THREADS)
# Contains artifact to be removed
delArtifacts = []
for gat in artifactList.keys():
groupId = gat.split(':')[0]
artifactId = gat.split(':')[1]
artifactType = gat.split(':')[2]
for priority in artifactList[gat].keys():
for version in artifactList[gat][priority].keys():
artifact = MavenArtifact(groupId, artifactId, artifactType, version)
pool.apply_async(
_artifactInRepos,
[self.config.excludedRepositories, artifact, priority, delArtifacts]
)
if not artifactList[gat][priority]:
del artifactList[gat][priority]
if not artifactList[gat]:
del artifactList[gat]
# Close the pool and wait for the workers to finnish
pool.close()
pool.join()
for artifact, priority in delArtifacts:
del artifactList[artifact.getGAT()][priority][artifact.version]
return artifactList
def depListToArtifactList(depList):
"""Convert the maven GAV to a URL relative path"""
regexComment = re.compile('#.*$')
#regexLog = re.compile('^\[\w*\]')
# Match pattern groupId:artifactId:type:[classifier:]version[:scope]
regexGAV = re.compile(
'(([\w\-.]+:){3}([\w\-.]+:)?([\d][\w\-.]+))(:[\w]*\S)?')
artifactList = []
for nextLine in depList:
nextLine = regexComment.sub('', nextLine)
nextLine = nextLine.strip()
gav = regexGAV.search(nextLine)
if gav:
artifactList.append(MavenArtifact.createFromGAV(gav.group(1)))
return artifactList
def generate_report(output, config, artifact_list, report_name):
"""
Generates report. The report consists of a summary page, groupId pages, artifactId pages and leaf artifact pages.
Summary page contains list of roots, list of BOMs, list of multi-version artifacts, links to all artifacts and list
of artifacts that do not match the BOM version. Each artifact has a separate page containing paths from roots to
the artifact with path explanation.
"""
multiversion_gas = dict()
malformed_versions = dict()
if os.path.exists(output):
logging.warn("Target report path %s exists. Deleting...", output)
shutil.rmtree(output)
os.makedirs(os.path.join(output, "pages"))
roots = []
for artifact_source in config.artifactSources:
if artifact_source["type"] == "dependency-graph":
roots.extend(artifact_source['top-level-gavs'])
groupids = dict()
version_pattern = re.compile("^.*[.-]redhat-[^.]+$")
for ga in artifact_list:
(groupid, artifactid) = ga.split(":")
priority_list = artifact_list[ga]
for priority in priority_list:
versions = priority_list[priority]
if versions:
groupids.setdefault(groupid, dict()).setdefault(artifactid, dict()).update(versions)
if len(groupids[groupid][artifactid]) > 1:
multiversion_gas.setdefault(groupid, dict())[artifactid] = groupids[groupid][artifactid]
for version in versions:
if not version_pattern.match(version):
malformed_versions.setdefault(groupid, dict()).setdefault(artifactid, dict())[version] = groupids[groupid][artifactid][version]
optional_artifacts = dict()
for groupid in groupids.keys():
artifactids = groupids[groupid]
for artifactid in artifactids.keys():
versions = artifactids[artifactid]
for version in versions.keys():
art_spec = versions[version]
ma = MavenArtifact.createFromGAV("%s:%s:%s" % (groupid, artifactid, version))
generate_artifact_page(ma, roots, art_spec.paths, art_spec.url, output, groupids, optional_artifacts)
generate_artifactid_page(groupid, artifactid, versions, output)
generate_groupid_page(groupid, artifactids, output)
generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts)
generate_css(output)
def _listDependencyGraph(self, aproxUrl, wsid, sourceKey, gavs):
"""
Loads maven artifacts from dependency graph.
:param aproxUrl: URL of the AProx instance
:param gavs: List of top level GAVs
:param wsid: workspace ID
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
from aprox_apis import AproxApi10
aprox = AproxApi10(aproxUrl)
deleteWS = False
if not wsid:
# Create workspace
ws = aprox.createWorkspace()
wsid = ws["id"]
deleteWS = True
# Resolve graph MANIFEST for GAVs
urlmap = aprox.urlmap(wsid, sourceKey, gavs,
self.configuration.allClassifiers)
# parse returned map
artifacts = {}
for gav in urlmap:
artifact = MavenArtifact.createFromGAV(gav)
groupId = artifact.groupId
artifactId = artifact.artifactId
version = artifact.version
filenames = urlmap[gav]["files"]
url = urlmap[gav]["repoUrl"]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifactId, version, filenames)
self._addArtifact(artifacts, groupId, artifactId, version,
extsAndClass, suffix, url)
# cleanup
if deleteWS:
aprox.deleteWorkspace(wsid)
return artifacts
def _listDependencies(self, repoUrls, gavs):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
it's repo root URL, or empty dictionary if something goes wrong.
"""
artifacts = {}
for gav in gavs:
artifact = MavenArtifact.createFromGAV(gav)
pomDir = 'poms'
fetched = False
for repoUrl in repoUrls:
pomUrl = repoUrl + '/' + artifact.getPomFilepath()
if fetchArtifact(pomUrl, pomDir):
fetched = True
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s",
gav)
continue
# Build dependency:list
mvnOutDir = "maven"
if not os.path.isdir(mvnOutDir):
os.makedirs(mvnOutDir)
mvnOutFilename = mvnOutDir + "/" + artifact.getBaseFilename() + "-maven.out"
with open(mvnOutFilename, "w") as mvnOutputFile:
retCode = call(['mvn', 'dependency:list', '-N', '-f',
pomDir + '/' + artifact.getPomFilename()], stdout=mvnOutputFile)
if retCode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s",
gav)
continue
# Parse GAVs from maven output
gavList = self._parseDepList(mvnOutFilename)
artifacts.update(self._listArtifacts(repoUrls, gavList))
return artifacts
def test_listMeadTagArtifacts(self):
config = configuration.Configuration()
config.allClassifiers = True
kojiUrl = "http://brewhub.devel.redhat.com/brewhub/"
tagName = "mead-import-maven"
downloadRootUrl = "http://download.devel.redhat.com/brewroot/packages/"
gavPatterns = ['org.apache.maven:maven-core:2.0.6']
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listMeadTagArtifacts(
kojiUrl, downloadRootUrl, tagName, gavPatterns)
expectedUrl = downloadRootUrl + "org.apache.maven-maven-core/2.0.6/1/maven/"
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]):
ArtifactSpec(expectedUrl, set(['javadoc', 'sources']))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def test_listMeadTagArtifacts(self):
config = configuration.Configuration()
config.allClassifiers = True
kojiUrl = "http://brewhub.devel.redhat.com/brewhub/"
tagName = "mead-import-maven"
downloadRootUrl = "http://download.devel.redhat.com/brewroot/packages/"
gavPatterns = [
'org.apache.maven:maven-core:2.0.6'
]
builder = artifact_list_builder.ArtifactListBuilder(config)
actualArtifacts = builder._listMeadTagArtifacts(kojiUrl, downloadRootUrl, tagName, gavPatterns)
expectedUrl = downloadRootUrl + "org.apache.maven-maven-core/2.0.6/1/maven/"
expectedArtifacts = {
MavenArtifact.createFromGAV(gavPatterns[0]): ArtifactSpec(expectedUrl, set(['javadoc', 'sources']))
}
self.assertEqualArtifactList(expectedArtifacts, actualArtifacts)
def _listDependencyGraph(self, aproxUrl, wsid, sourceKey, gavs):
"""
Loads maven artifacts from dependency graph.
:param aproxUrl: URL of the AProx instance
:param gavs: List of top level GAVs
:param wsid: workspace ID
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
from aprox_apis import AproxApi10
aprox = AproxApi10(aproxUrl)
deleteWS = False
if not wsid:
# Create workspace
ws = aprox.createWorkspace()
wsid = ws["id"]
deleteWS = True
# Resolve graph MANIFEST for GAVs
urlmap = aprox.urlmap(wsid, sourceKey, gavs, self.configuration.allClassifiers)
# parse returned map
artifacts = {}
for gav in urlmap:
artifact = MavenArtifact.createFromGAV(gav)
groupId = artifact.groupId
artifactId = artifact.artifactId
version = artifact.version
filenames = urlmap[gav]["files"]
url = urlmap[gav]["repoUrl"]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)
# cleanup
if deleteWS:
aprox.deleteWorkspace(wsid)
return artifacts
def generateArtifactList(options):
"""
Generates artifact "list" from sources defined in the given configuration in options. The result
is dictionary with following structure:
<repo url> (string)
L artifacts (list of MavenArtifact)
"""
artifactList = _generateArtifactList(options)
#build sane structure - url to MavenArtifact list
urlToMAList = {}
for gat in artifactList:
prioList = artifactList[gat]
for priority in prioList:
verList = prioList[priority]
for version in verList:
url = verList[version]
urlToMAList.setdefault(url, []).append(MavenArtifact.createFromGAV(gat + ":" + version))
return urlToMAList
def _listArtifacts(self, urls, gavs):
"""
Loads maven artifacts from list of GAVs and tries to locate the artifacts in one of the
specified repositories.
:param urls: repository URLs where the given GAVs can be located
:param gavs: List of GAVs
:returns: Dictionary where index is MavenArtifact object and value is it's repo root URL.
"""
artifacts = {}
for gav in gavs:
artifact = MavenArtifact.createFromGAV(gav)
for url in urls:
gavUrl = url + '/' + artifact.getDirPath()
if mrbutils.urlExists(gavUrl):
artifacts[artifact] = url
break
if not artifact in artifacts:
logging.warning('artifact %s not found in any url!', artifact)
return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:param recursive: runs dependency:list recursively using the previously discovered dependencies if True
:param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None
to use Maven's default
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
if include_scope:
args.append("-DincludeScope=%s" % include_scope)
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.isAllClassifiers():
resultingArtifacts = {}
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn("Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if artifact.artifactType in extsAndClass:
self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
else:
if files:
logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s",
artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files))
else:
logging.warn("An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
newArtifacts = resultingArtifacts
artifacts.update(newArtifacts)
return artifacts
def generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts):
html = ("<html><head><title>Repository {report_name}</title>" + \
"<link rel=\"stylesheet\" type=\"text/css\" href=\"http://code.jquery.com/ui/1.11.4/themes/smoothness/jquery-ui.css\">" + \
"<script src=\"http://code.jquery.com/jquery-1.10.2.js\"></script>" + \
"<script src=\"http://code.jquery.com/ui/1.11.4/jquery-ui.js\"></script>" + \
"<link rel=\"stylesheet\" type=\"text/css\" href=\"pages/style.css\">" + \
"<script>$(function() {script});</script></head><body>" + \
"<div class=\"artifact\"><h1>{report_name}</h1><div id=\"tabs\">" + \
"<ul>" + \
"<li><a href=\"#tab-definition\">Artifact sources</a></li>" + \
"<li><a href=\"#tab-multi-versioned-artifacts\">Multi-versioned artifacts</a></li>" + \
"<li><a href=\"#tab-malformed-versions\">Malformed versions</a></li>" + \
"<li><a href=\"#tab-optional-artifacts\">Optional artifacts</a></li>" + \
"<li><a href=\"#tab-all-artifacts\">All artifacts</a></li>" + \
"</ul>" + \
"<div id=\"tab-definition\">" + \
"<h2>Artifact sources</h2>").format(report_name=report_name, script="{ $(\"#tabs\").tabs(); }")
examples = ""
i = 1
for artifact_source in config.artifactSources:
html += "<div class=\"artifact-source\">"
if artifact_source["type"] == "dependency-graph":
html += "<h3>Dependency graph #%i</h3><h4>Roots</h4><ul>" % i
i += 1
for root in sorted(artifact_source['top-level-gavs']):
ma = MavenArtifact.createFromGAV(root)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
if ma.is_example():
examples += "<li class=\"error\"><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
html += "<li><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
if ma.is_example():
examples += "<li class=\"example\">{gid}{colon}{aid}{colon}{ver}</li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
html += "<li class=\"error\">{gid}{colon}{aid}{colon}{ver}</li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
html += examples + "</ul><h4>BOMs</h4><ul>"
if len(artifact_source['injected-boms']):
for bom in artifact_source['injected-boms']:
ma = MavenArtifact.createFromGAV(bom)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
html += "<li><a href=\"pages/artifact_version_{gid}${aid}${ver}.html\">{gid}{colon}{aid}{colon}{ver}</a></li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
html += "<li><span class=\"error\">{gid}{colon}{aid}{colon}{ver}</span></li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
html += "<li><em>none</em></li>"
if len(artifact_source['excluded-subgraphs']):
html += "</ul><h4>Excluded subgraphs</h4><ul>"
for exclusion in artifact_source['excluded-subgraphs']:
ma = MavenArtifact.createFromGAV(exclusion)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
html += "<li><a class=\"error\" href=\"pages/artifact_version_{gid}${aid}${ver}.html\">" + \
"{gid}{colon}{aid}{colon}{ver}</a></li>".format(gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
else:
html += "<li class=\"excluded\">{gid}{colon}{aid}{colon}{ver}</li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML)
if artifact_source['preset'] in ["sob-build", "scope-with-embedded", "requires", "managed-sob-build"]:
contents = "runtime dependencies"
elif artifact_source['preset'] in ["sob", "build-env", "build-requires", "br", "managed-sob"]:
contents = "build-time dependencies"
else:
contents = artifact_source['preset']
html += "</ul><h4>Contents</h4><ul><li>%s</li></ul>" % contents
else:
html += artifact_source["type"]
html += "</div>"
html += "<div class=\"artifact-source\"><h3>Global</h3><h4>Excluded GA(TC)V patterns</h4><ul>"
if len(config.excludedGAVs):
for excluded_pattern in config.excludedGAVs:
html += "<li class=\"excluded\">%s</li>" % excluded_pattern
else:
html += "<li><em>none</em></li>"
html += "</ul><h4>Excluded repositories</h4><ul>"
if len(config.excludedRepositories):
for excluded_repo in config.excludedRepositories:
html += "<li class=\"excluded\"><a href=\"%(url)s\">%(url)s</a></li>" % {"url": excluded_repo}
else:
html += "<li><em>none</em></li>"
html += "</ul></div>"
html += "</div>\n<div id=\"tab-multi-versioned-artifacts\"><h2>Multi-versioned artifacts</h2><ul>"
for groupid in sorted(multiversion_gas.keys()):
artifactids = multiversion_gas[groupid]
for artifactid in sorted(artifactids.keys()):
html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
"<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>" + \
"</li><ul>").format(artifactid=artifactid, groupid=groupid, colon=COLON_HTML)
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += "<li><a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>".format(
version=version, gav_filename=gav.replace(":", "$"))
html += "</ul>"
html += "</ul></div>\n<div id=\"tab-malformed-versions\"><h2>Malformed versions</h2><ul>"
for groupid in sorted(malformed_versions.keys()):
artifactids = malformed_versions[groupid]
for artifactid in sorted(artifactids.keys()):
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
"<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>{colon}" + \
"<a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>").format(artifactid=artifactid, groupid=groupid,
version=version,
gav_filename=gav.replace(":", "$"),
colon=COLON_HTML)
html += "</ul></div>\n<div id=\"tab-optional-artifacts\"><h2>Optional artifacts</h2><h3>Direct optionals</h3><ul>"
for ma in sorted(optional_artifacts.keys()):
if optional_artifacts[ma]:
gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\" class=\"optional\">{groupid}</a>{colon}" + \
"<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\" class=\"optional\">{artifactid}</a>{colon}" + \
"<a href=\"pages/artifact_version_{gav_filename}.html\" class=\"optional\">{version}</a></li>").format(groupid=ma.groupId,
artifactid=ma.artifactId,
version=ma.version,
gav_filename=gav_filename,
colon=COLON_HTML)
html += "</ul><h3>Transitive optionals</h3><ul>"
for ma in sorted(optional_artifacts.keys()):
if not optional_artifacts[ma]:
gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\" class=\"optional\">{groupid}</a>{colon}" + \
"<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\" class=\"optional\">{artifactid}</a>{colon}" + \
"<a href=\"pages/artifact_version_{gav_filename}.html\" class=\"optional\">{version}</a></li>").format(groupid=ma.groupId,
artifactid=ma.artifactId,
version=ma.version,
gav_filename=gav_filename,
colon=COLON_HTML)
html += "</ul></div>\n<div id=\"tab-all-artifacts\"><h2>All artifacts</h2><ul>"
for groupid in sorted(groupids.keys()):
artifactids = groupids[groupid]
for artifactid in sorted(artifactids.keys()):
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += ("<li><a href=\"pages/groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>{colon}" + \
"<a href=\"pages/artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>{colon}" + \
"<a href=\"pages/artifact_version_{gav_filename}.html\">{version}</a></li>").format(groupid=groupid, artifactid=artifactid,
version=version, gav_filename=gav.replace(":", "$"), colon=COLON_HTML)
html += "</ul></div></div></div></body></html>"
with open(os.path.join(output, "index.html"), "w") as htmlfile:
htmlfile.write(html)
def _listDependencies(self, repoUrls, gavs):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
it's repo root URL, or empty dictionary if something goes wrong.
"""
artifacts = {}
for gav in gavs:
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
with open(settingsFile, 'w') as settings:
settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL)
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
logging.debug("Running Maven:\n %s", " ".join(args))
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
newArtifacts = self._listArtifacts(repoUrls, gavList)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
out = self._lftpFind(spec.url + artifact.getDirPath())
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(
repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s",
gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = [
'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile,
'-f', pomFilename, '-s', settingsFile
]
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning(
"Maven failed to finish with success. Skipping artifact %s",
gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn(
"Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
if artifact.artifactType in extsAndClass:
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId,
artifact.artifactId,
artifact.version, extsAndClass,
suffix, spec.url)
else:
if files:
logging.warn(
"Main artifact is missing in filelist listed from %s. Files were:\n%s",
spec.url + artifact.getDirPath(),
"\n".join(files))
else:
logging.warn(
"An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
artifacts.update(newArtifacts)
return artifacts
def _listDependencyGraph(self, indyUrl, wsid, sourceKey, gavs, excludedSources=[], excludedSubgraphs=[],
preset="requires", mutator=None, patcherIds=[], injectedBOMs=[], analyze=False):
"""
Loads maven artifacts from dependency graph.
:param indyUrl: URL of the Indy instance
:param wsid: workspace ID
:param sourceKey: the Indy artifact source key, consisting of the source type and
its name of the form <{repository|deploy|group}:<name>>
:param gavs: List of top level GAVs
:param excludedSources: list of excluded sources' keys
:param excludedSubgraphs: list of artifacts' GAVs which we want to exclude along with their subgraphs
:param preset: preset used while creating the urlmap
:param mutator: mutator used for dependency graph mutation
:param patcherIds: list of patcher ID strings for Indy
:param injectedBOMs: list of injected BOMs used with dependency management injection
Maven extension
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
indy = IndyApi(indyUrl)
if not preset:
preset = "requires" # only runtime dependencies
if analyze and not wsid:
_wsid = "temp"
else:
_wsid = wsid
# Resolve graph MANIFEST for GAVs
if self.configuration.useCache:
urlmap = indy.urlmap(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
else:
urlmap = indy.urlmap_nocache(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
# parse returned map
artifacts = {}
if "projects" in urlmap:
urlmap = urlmap["projects"]
for gav in urlmap:
artifact = MavenArtifact.createFromGAV(gav)
groupId = artifact.groupId
artifactId = artifact.artifactId
version = artifact.version
filenames = urlmap[gav]["files"]
url = urlmap[gav]["repoUrl"]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)
if analyze:
gas = []
for ma in artifacts.keys():
ga = ma.getGA()
if not ga in gas:
gas.append(ga)
try:
if self.configuration.useCache:
path_dict = indy.paths(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs, preset,
mutator, patcherIds, injectedBOMs, False)
else:
path_dict = indy.paths_nocache(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs,
preset, mutator, patcherIds, injectedBOMs, False)
except RuntimeError:
path_dict = dict()
if "projects" in path_dict:
path_dict = path_dict["projects"]
if path_dict:
for ma in artifacts.keys():
for key in path_dict.keys():
key_gav = key
if key.count(":") > 2:
key_split = key.split(":")
key_gav = ":".join([key_split[0], key_split[1], key_split[-1]])
if ma.getGAV() == key_gav:
gav_paths = path_dict[key]
if type(gav_paths) is dict:
gav_paths = gav_paths["paths"]
for gav_path in gav_paths:
direct = True
if type(gav_path) is dict:
gav_path = gav_path["pathParts"]
rel_path = []
for gav_rel in gav_path:
if "inherited" in gav_rel and gav_rel["inherited"]:
direct = False
break
declaring = MavenArtifact.createFromGAV(gav_rel["declaring"])
target = MavenArtifact.createFromGAV(maven_repo_util.gatvc_to_gatcv(gav_rel["target"]))
rel_type = gav_rel["type"] if "type" in gav_rel else gav_rel["rel"]
if rel_type == "DEPENDENCY":
extra = gav_rel["scope"]
if "optional" in gav_rel and gav_rel["optional"]:
extra = "%s %s" % (extra, "optional")
rel = ArtifactRelationship(declaring, target, rel_type, extra)
elif rel_type == "PLUGIN_DEP":
rel = ArtifactRelationship(declaring, target, rel_type, gav_rel["plugin"])
elif rel_type == "BOM":
if "mixin" in gav_rel and gav_rel["mixin"]:
direct = False
break
rel = ArtifactRelationship(declaring, target, rel_type)
else:
rel = ArtifactRelationship(declaring, target, rel_type)
rel_path.append(rel)
if direct:
artifacts[ma].add_path(rel_path)
for ma in artifacts.keys():
if not artifacts[ma].paths and ma.getGAV() not in gavs:
# create artificial unknown paths from all roots to current artifact
for root in gavs:
if root != ma.getGAV():
rel_path = [ArtifactRelationship(MavenArtifact.createFromGAV(root), None, None),
ArtifactRelationship(None, ma, None)]
artifacts[ma].add_path(rel_path)
if not wsid:
try:
indy.deleteWorkspace(_wsid)
except Exception as err:
logging.warning("Workspace deletion failed: %s" % str(err))
return artifacts
def _filterExcludedTypes(self, artifactList):
'''
Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
not-excluded artifact types only.
:param artifactList: artifactList to be filtered.
:param exclTypes: list of excluded types
:returns: artifactList without artifacts that matched specified types and had no other main types.
'''
logging.debug("Filtering artifacts with excluded types.")
regExps = maven_repo_util.getRegExpsFromStrings(
self.config.gatcvWhitelist)
exclTypes = self.config.excludedTypes
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artSpec = artifactList[ga][priority][version]
for artType in list(artSpec.artTypes.keys()):
if artType in exclTypes:
artTypeObj = artSpec.artTypes[artType]
classifiers = artTypeObj.classifiers
(groupId, artifactId) = ga.split(':')
for classifier in list(classifiers):
art = MavenArtifact(groupId, artifactId,
artType, version,
classifier)
gatcv = art.getGATCV()
if not maven_repo_util.somethingMatch(
regExps, gatcv):
logging.debug(
"Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
"excluded type.", classifier, ga,
artType, version, priority)
classifiers.remove(classifier)
else:
logging.debug(
"Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
"whitelist pattern.", ga, artType,
classifier, version, priority)
if not classifiers:
logging.debug(
"Dropping %s:%s:%s from priority %i because of no classifier left.",
ga, artType, version, priority)
del (artSpec.artTypes[artType])
noMain = True
for artType in artSpec.artTypes.keys():
artTypeObj = artSpec.artTypes[artType]
if artTypeObj.mainType:
noMain = False
break
if not artSpec.artTypes or noMain:
if noMain:
logging.debug(
"Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
else:
logging.debug(
"Dropping GAV %s:%s from priority %i because of no artifact type left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug(
"Dropping GA %s from priority %i because of no version left.",
ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.",
ga)
del artifactList[ga]
return artifactList
def _listDependencyGraph(self, cartoUrl, wsid, sourceKey, gavs, excludedSources=[], excludedSubgraphs=[],
preset="requires", mutator=None, patcherIds=[], injectedBOMs=[], analyze=False):
"""
Loads maven artifacts from dependency graph.
:param cartoUrl: URL of the Cartographer instance
:param wsid: workspace ID
:param sourceKey: the Cartographer artifact source key, consisting of the source type and
its name of the form <{repository|deploy|group}:<name>>
:param gavs: List of top level GAVs
:param excludedSources: list of excluded sources' keys
:param excludedSubgraphs: list of artifacts' GAVs which we want to exclude along with their subgraphs
:param preset: preset used while creating the urlmap
:param mutator: mutator used for dependency graph mutation
:param patcherIds: list of patcher ID strings for Cartographer
:param injectedBOMs: list of injected BOMs used with dependency management injection
Maven extension
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
carto = CartoClient(cartoUrl)
if not preset:
preset = "requires" # only runtime dependencies
if analyze and not wsid:
_wsid = "temp"
else:
_wsid = wsid
# Resolve graph MANIFEST for GAVs
if self.configuration.useCache:
urlmap = carto.urlmap(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
else:
urlmap = carto.urlmap_nocache(_wsid, sourceKey, gavs, self.configuration.addClassifiers, excludedSources,
excludedSubgraphs, preset, mutator, patcherIds, injectedBOMs)
# parse returned map
artifacts = {}
if "projects" in urlmap:
urlmap = urlmap["projects"]
for gav in urlmap:
artifact = MavenArtifact.createFromGAV(gav)
groupId = artifact.groupId
artifactId = artifact.artifactId
version = artifact.version
filenames = urlmap[gav]["files"]
url = urlmap[gav]["repoUrl"]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
self._addArtifact(artifacts, groupId, artifactId, version, extsAndClass, suffix, url)
if analyze:
gas = []
for ma in artifacts.keys():
ga = ma.getGA()
if not ga in gas:
gas.append(ga)
try:
if self.configuration.useCache:
path_dict = carto.paths(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs, preset,
mutator, patcherIds, injectedBOMs, False)
else:
path_dict = carto.paths_nocache(_wsid, sourceKey, gavs, gas, excludedSources, excludedSubgraphs,
preset, mutator, patcherIds, injectedBOMs, False)
except RuntimeError:
path_dict = dict()
if "projects" in path_dict:
path_dict = path_dict["projects"]
if path_dict:
for ma in artifacts.keys():
for key in path_dict.keys():
key_gav = key
if key.count(":") > 2:
key_split = key.split(":")
key_gav = ":".join([key_split[0], key_split[1], key_split[-1]])
if ma.getGAV() == key_gav:
gav_paths = path_dict[key]
if type(gav_paths) is dict:
gav_paths = gav_paths["paths"]
for gav_path in gav_paths:
direct = True
if type(gav_path) is dict:
gav_path = gav_path["pathParts"]
rel_path = []
for gav_rel in gav_path:
if "inherited" in gav_rel and gav_rel["inherited"]:
direct = False
break
declaring = MavenArtifact.createFromGAV(gav_rel["declaring"])
target = MavenArtifact.createFromGAV(maven_repo_util.gatvc_to_gatcv(gav_rel["target"]))
rel_type = gav_rel["type"] if "type" in gav_rel else gav_rel["rel"]
if rel_type == "DEPENDENCY":
extra = gav_rel["scope"]
if "optional" in gav_rel and gav_rel["optional"]:
extra = "%s %s" % (extra, "optional")
rel = ArtifactRelationship(declaring, target, rel_type, extra)
elif rel_type == "PLUGIN_DEP":
rel = ArtifactRelationship(declaring, target, rel_type, gav_rel["plugin"])
elif rel_type == "BOM":
if "mixin" in gav_rel and gav_rel["mixin"]:
direct = False
break
rel = ArtifactRelationship(declaring, target, rel_type)
else:
rel = ArtifactRelationship(declaring, target, rel_type)
rel_path.append(rel)
if direct:
artifacts[ma].add_path(rel_path)
for ma in artifacts.keys():
if not artifacts[ma].paths and ma.getGAV() not in gavs:
# create artificial unknown paths from all roots to current artifact
for root in gavs:
if root != ma.getGAV():
rel_path = [ArtifactRelationship(MavenArtifact.createFromGAV(root), None, None),
ArtifactRelationship(None, ma, None)]
artifacts[ma].add_path(rel_path)
if not wsid:
try:
carto.deleteWorkspace(_wsid)
except Exception as err:
logging.warning("Workspace deletion failed: %s" % str(err))
return artifacts
def generate_summary(config, groupids, multiversion_gas, malformed_versions, output, report_name, optional_artifacts):
html = (
"<html><head><title>Repository {report_name}</title>"
+ '<link rel="stylesheet" type="text/css" href="http://code.jquery.com/ui/1.11.4/themes/smoothness/jquery-ui.css">'
+ '<script src="http://code.jquery.com/jquery-1.10.2.js"></script>'
+ '<script src="http://code.jquery.com/ui/1.11.4/jquery-ui.js"></script>'
+ '<link rel="stylesheet" type="text/css" href="pages/style.css">'
+ "<script>$(function() {script});</script></head><body>"
+ '<div class="artifact"><h1>{report_name}</h1><div id="tabs">'
+ "<ul>"
+ '<li><a href="#tab-definition">Artifact sources</a></li>'
+ '<li><a href="#tab-multi-versioned-artifacts">Multi-versioned artifacts</a></li>'
+ '<li><a href="#tab-malformed-versions">Malformed versions</a></li>'
+ '<li><a href="#tab-optional-artifacts">Optional artifacts</a></li>'
+ '<li><a href="#tab-all-artifacts">All artifacts</a></li>'
+ "</ul>"
+ '<div id="tab-definition">'
+ "<h2>Artifact sources</h2>"
).format(report_name=report_name, script='{ $("#tabs").tabs(); }')
examples = ""
i = 1
for artifact_source in config.artifactSources:
html += '<div class="artifact-source">'
if artifact_source["type"] == "dependency-graph":
html += "<h3>Dependency graph #%i</h3><h4>Roots</h4><ul>" % i
i += 1
for root in sorted(artifact_source["top-level-gavs"]):
ma = MavenArtifact.createFromGAV(root)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
if ma.is_example():
examples += '<li class="error"><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
else:
html += '<li><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
else:
if ma.is_example():
examples += '<li class="example">{gid}{colon}{aid}{colon}{ver}</li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
else:
html += '<li class="error">{gid}{colon}{aid}{colon}{ver}</li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
html += examples + "</ul><h4>BOMs</h4><ul>"
if len(artifact_source["injected-boms"]):
for bom in artifact_source["injected-boms"]:
ma = MavenArtifact.createFromGAV(bom)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
html += '<li><a href="pages/artifact_version_{gid}${aid}${ver}.html">{gid}{colon}{aid}{colon}{ver}</a></li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
else:
html += '<li><span class="error">{gid}{colon}{aid}{colon}{ver}</span></li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
else:
html += "<li><em>none</em></li>"
if len(artifact_source["excluded-subgraphs"]):
html += "</ul><h4>Excluded subgraphs</h4><ul>"
for exclusion in artifact_source["excluded-subgraphs"]:
ma = MavenArtifact.createFromGAV(exclusion)
gid = ma.groupId
aid = ma.artifactId
ver = ma.version
if gid in groupids.keys() and aid in groupids[gid].keys() and ver in groupids[gid][aid]:
html += (
'<li><a class="error" href="pages/artifact_version_{gid}${aid}${ver}.html">'
+ "{gid}{colon}{aid}{colon}{ver}</a></li>".format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
)
else:
html += '<li class="excluded">{gid}{colon}{aid}{colon}{ver}</li>'.format(
gid=gid, aid=aid, ver=ver, colon=COLON_HTML
)
if artifact_source["preset"] in ["sob-build", "scope-with-embedded", "requires", "managed-sob-build"]:
contents = "runtime dependencies"
elif artifact_source["preset"] in ["sob", "build-env", "build-requires", "br", "managed-sob"]:
contents = "build-time dependencies"
else:
contents = artifact_source["preset"]
html += "</ul><h4>Contents</h4><ul><li>%s</li></ul>" % contents
else:
html += artifact_source["type"]
html += "</div>"
html += '<div class="artifact-source"><h3>Global</h3><h4>Excluded GA(TC)V patterns</h4><ul>'
if len(config.excludedGAVs):
for excluded_pattern in config.excludedGAVs:
html += '<li class="excluded">%s</li>' % excluded_pattern
else:
html += "<li><em>none</em></li>"
html += "</ul><h4>Excluded repositories</h4><ul>"
if len(config.excludedRepositories):
for excluded_repo in config.excludedRepositories:
html += '<li class="excluded"><a href="%(url)s">%(url)s</a></li>' % {"url": excluded_repo}
else:
html += "<li><em>none</em></li>"
html += "</ul></div>"
html += '</div>\n<div id="tab-multi-versioned-artifacts"><h2>Multi-versioned artifacts</h2><ul>'
for groupid in sorted(multiversion_gas.keys()):
artifactids = multiversion_gas[groupid]
for artifactid in sorted(artifactids.keys()):
html += (
'<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
+ '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>'
+ "</li><ul>"
).format(artifactid=artifactid, groupid=groupid, colon=COLON_HTML)
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += '<li><a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'.format(
version=version, gav_filename=gav.replace(":", "$")
)
html += "</ul>"
html += '</ul></div>\n<div id="tab-malformed-versions"><h2>Malformed versions</h2><ul>'
for groupid in sorted(malformed_versions.keys()):
artifactids = malformed_versions[groupid]
for artifactid in sorted(artifactids.keys()):
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += (
'<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
+ '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>{colon}'
+ '<a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'
).format(
artifactid=artifactid,
groupid=groupid,
version=version,
gav_filename=gav.replace(":", "$"),
colon=COLON_HTML,
)
html += '</ul></div>\n<div id="tab-optional-artifacts"><h2>Optional artifacts</h2><h3>Direct optionals</h3><ul>'
for ma in sorted(optional_artifacts.keys()):
if optional_artifacts[ma]:
gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
html += (
'<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}" class="optional">{groupid}</a>{colon}'
+ '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}" class="optional">{artifactid}</a>{colon}'
+ '<a href="pages/artifact_version_{gav_filename}.html" class="optional">{version}</a></li>'
).format(
groupid=ma.groupId,
artifactid=ma.artifactId,
version=ma.version,
gav_filename=gav_filename,
colon=COLON_HTML,
)
html += "</ul><h3>Transitive optionals</h3><ul>"
for ma in sorted(optional_artifacts.keys()):
if not optional_artifacts[ma]:
gav_filename = "%s$%s$%s" % (ma.groupId, ma.artifactId, ma.version)
html += (
'<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}" class="optional">{groupid}</a>{colon}'
+ '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}" class="optional">{artifactid}</a>{colon}'
+ '<a href="pages/artifact_version_{gav_filename}.html" class="optional">{version}</a></li>'
).format(
groupid=ma.groupId,
artifactid=ma.artifactId,
version=ma.version,
gav_filename=gav_filename,
colon=COLON_HTML,
)
html += '</ul></div>\n<div id="tab-all-artifacts"><h2>All artifacts</h2><ul>'
for groupid in sorted(groupids.keys()):
artifactids = groupids[groupid]
for artifactid in sorted(artifactids.keys()):
artifacts = artifactids[artifactid]
for version in sorted(artifacts.keys()):
gav = "%s:%s:%s" % (groupid, artifactid, version)
html += (
'<li><a href="pages/groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>{colon}'
+ '<a href="pages/artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>{colon}'
+ '<a href="pages/artifact_version_{gav_filename}.html">{version}</a></li>'
).format(
groupid=groupid,
artifactid=artifactid,
version=version,
gav_filename=gav.replace(":", "$"),
colon=COLON_HTML,
)
html += "</ul></div></div></div></body></html>"
with open(os.path.join(output, "index.html"), "w") as htmlfile:
htmlfile.write(html)