def _listRepository(self, repoUrls, gavPatterns):
"""
Loads maven artifacts from a repository.
:param repoUrl: repository URL (local or remote, supported are [file://], http:// and
https:// urls)
:param gavPatterns: list of patterns to filter by GAV
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
prefixes = self._getPrefixes(gavPatterns)
artifacts = {}
for repoUrl in reversed(repoUrls):
urlWithSlash = maven_repo_util.slashAtTheEnd(repoUrl)
protocol = maven_repo_util.urlProtocol(urlWithSlash)
if protocol == 'file':
for prefix in prefixes:
artifacts.update(self._listLocalRepository(urlWithSlash[7:], prefix))
elif protocol == '':
for prefix in prefixes:
artifacts.update(self._listLocalRepository(urlWithSlash, prefix))
elif protocol == 'http' or protocol == 'https':
for prefix in prefixes:
artifacts.update(self._listRemoteRepository(urlWithSlash, prefix))
else:
raise "Invalid protocol!", protocol
artifacts = self._filterArtifactsByPatterns(artifacts, gavPatterns)
logging.debug("Found %d artifacts", len(artifacts))
return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode,
mkdirLock, filesetLock, fileset, errors):
"""Download artifact from a remote repository."""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath())
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode,
True, True, filesetLock, fileset)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact,
str(ex))
errors.put(ex)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is it's repo root URL.
"""
kojiSession = koji.ClientSession(kojiUrl)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
gavsWithExts = {}
for artifact in kojiArtifacts:
# FIXME: This isn't reliable as file extension is not equal to
# maven type, e.g. jar != ejb
artifactType = re.search('.*\.(.+)$', artifact['filename']).group(1)
gavUrl = mrbutils.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\
+ artifact['build_version'] + '/' + artifact['build_release'] + '/maven/'
gavu = (artifact['group_id'], artifact['artifact_id'], artifact['version'], gavUrl)
gavsWithExts.setdefault(gavu, []).append(artifactType)
artifacts = {}
for gavu in gavsWithExts:
if len(gavsWithExts[gavu]) > 1:
gavsWithExts[gavu].remove("pom")
for ext in gavsWithExts[gavu]:
mavenArtifact = MavenArtifact(gavu[0], gavu[1], ext, gavu[2])
artifacts[mavenArtifact] = gavu[3]
return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def _listRepository(self, repoUrls, gavPatterns):
"""
Loads maven artifacts from a repository.
:param repoUrl: repository URL (local or remote, supported are [file://], http:// and
https:// urls)
:param gavPatterns: list of patterns to filter by GAV
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
prefixes = self._getPrefixes(gavPatterns)
artifacts = {}
for repoUrl in reversed(repoUrls):
urlWithSlash = maven_repo_util.slashAtTheEnd(repoUrl)
protocol = maven_repo_util.urlProtocol(urlWithSlash)
if protocol == 'file':
for prefix in prefixes:
artifacts.update(
self._listLocalRepository(urlWithSlash[7:], prefix))
elif protocol == '':
for prefix in prefixes:
artifacts.update(
self._listLocalRepository(urlWithSlash, prefix))
elif protocol == 'http' or protocol == 'https':
for prefix in prefixes:
artifacts.update(
self._listRemoteRepository(urlWithSlash, prefix))
else:
raise "Invalid protocol!", protocol
artifacts = self._filterArtifactsByPatterns(artifacts, gavPatterns)
logging.debug("Found %d artifacts", len(artifacts))
return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers,
checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl,
artifactLocalPath,
checksumMode,
exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir,
artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl,
artifactPomLocalPath,
checksumMode,
exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(
classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir,
artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(artifactClassifierUrl,
artifactClassifierLocalPath,
checksumMode,
exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact,
str(ex))
errors.put(ex)
def _isArtifactInRepos(repositories, artifact):
"""
Returns True if specified artifact exists in at least one repositori from specified list.
:param repositories: list of repository urls
:param artifact: searched MavenArtifact
:returns: True if specified artifact exists in at least one of the repositories.
"""
for repository in repositories:
url = maven_repo_util.slashAtTheEnd(repository) + artifact.getDirPath()
if maven_repo_util.urlExists(url):
return True
return False
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
import koji
kojiSession = koji.ClientSession(kojiUrl)
logging.debug("Getting latest maven artifacts from tag %s.", tagName)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
filenameDict = {}
for artifact in kojiArtifacts:
groupId = artifact['group_id']
artifactId = artifact['artifact_id']
version = artifact['version']
gavUrl = "%s%s/%s/%s/maven/" % (maven_repo_util.slashAtTheEnd(downloadRootUrl), artifact['build_name'],
artifact['build_version'], artifact['build_release'])
gavu = (groupId, artifactId, version, gavUrl)
filename = artifact['filename']
filenameDict.setdefault(gavu, []).append(filename)
gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} }
suffixes = {} # { (g,a,v,url): suffix }
for gavu in filenameDict:
artifactId = gavu[1]
version = gavu[2]
filenames = filenameDict[gavu]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
if extsAndClass:
gavuExtClass[gavu] = {}
self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass)
if suffix is not None:
suffixes[gavu] = suffix
artifacts = {}
for gavu in gavuExtClass:
self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3])
if gavPatterns:
logging.debug("Filtering artifacts contained in the tag by GAV patterns list.")
return self._filterArtifactsByPatterns(artifacts, gavPatterns, None)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
import koji
kojiSession = koji.ClientSession(kojiUrl)
logging.debug("Getting latest maven artifacts from tag %s.", tagName)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
filenameDict = {}
for artifact in kojiArtifacts:
groupId = artifact['group_id']
artifactId = artifact['artifact_id']
version = artifact['version']
gavUrl = "%s%s/%s/%s/maven/" % (maven_repo_util.slashAtTheEnd(downloadRootUrl), artifact['build_name'],
artifact['build_version'], artifact['build_release'])
gavu = (groupId, artifactId, version, gavUrl)
filename = artifact['filename']
filenameDict.setdefault(gavu, []).append(filename)
gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} }
suffixes = {} # { (g,a,v,url): suffix }
for gavu in filenameDict:
artifactId = gavu[1]
version = gavu[2]
filenames = filenameDict[gavu]
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames)
if extsAndClass:
gavuExtClass[gavu] = {}
self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass)
if suffix is not None:
suffixes[gavu] = suffix
artifacts = {}
for gavu in gavuExtClass:
self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3])
if gavPatterns:
logging.debug("Filtering artifacts contained in the tag by GAV patterns list.")
return self._filterArtifactsByPatterns(artifacts, gavPatterns, None)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName,
gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
import koji
kojiSession = koji.ClientSession(kojiUrl)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} }
suffixes = {} # { (g,a,v,url): suffix }
for artifact in kojiArtifacts:
groupId = artifact['group_id']
artifactId = artifact['artifact_id']
version = artifact['version']
filename = artifact['filename']
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifactId, version, [filename])
if extsAndClass:
gavUrl = maven_repo_util.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\
+ artifact['build_version'] + '/' + artifact['build_release'] + '/maven/'
gavu = (groupId, artifactId, version, gavUrl)
gavuExtClass.setdefault(gavu, {})
self._updateExtensionsAndClassifiers(gavuExtClass[gavu],
extsAndClass)
if suffix is not None and (gavu not in suffixes
or suffixes[gavu] < suffix):
suffixes[gavu] = suffix
artifacts = {}
for gavu in gavuExtClass:
self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2],
gavuExtClass[gavu], suffixes.get(gavu), gavu[3])
return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns):
"""
Loads maven artifacts from koji (brew/mead).
:param kojiUrl: Koji/Brew/Mead URL
:param downloadRootUrl: Download root URL of the artifacts
:param tagName: Koji/Brew/Mead tag name
:returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its
repo root URL.
"""
import koji
kojiSession = koji.ClientSession(kojiUrl)
kojiArtifacts = kojiSession.getLatestMavenArchives(tagName)
gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} }
suffixes = {} # { (g,a,v,url): suffix }
for artifact in kojiArtifacts:
groupId = artifact['group_id']
artifactId = artifact['artifact_id']
version = artifact['version']
filename = artifact['filename']
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, [filename])
if extsAndClass:
gavUrl = maven_repo_util.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\
+ artifact['build_version'] + '/' + artifact['build_release'] + '/maven/'
gavu = (groupId, artifactId, version, gavUrl)
gavuExtClass.setdefault(gavu, {})
self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass)
if suffix is not None and (gavu not in suffixes or suffixes[gavu] < suffix):
suffixes[gavu] = suffix
artifacts = {}
for gavu in gavuExtClass:
self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3])
return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors):
"""Download artifact from a remote repository."""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath())
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, True, True, filesetLock, fileset)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact, str(ex))
errors.put(ex)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir, artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(
artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact, str(ex))
errors.put(ex)
def __init__(self, carto_url):
self._carto_url = slashAtTheEnd(carto_url)
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:param recursive: runs dependency:list recursively using the previously discovered dependencies if True
:param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None
to use Maven's default
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
if include_scope:
args.append("-DincludeScope=%s" % include_scope)
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.isAllClassifiers():
resultingArtifacts = {}
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn("Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if artifact.artifactType in extsAndClass:
self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
else:
if files:
logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s",
artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files))
else:
logging.warn("An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
newArtifacts = resultingArtifacts
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
it's repo root URL, or empty dictionary if something goes wrong.
"""
artifacts = {}
for gav in gavs:
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
with open(settingsFile, 'w') as settings:
settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL)
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
logging.debug("Running Maven:\n %s", " ".join(args))
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
newArtifacts = self._listArtifacts(repoUrls, gavList)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
out = self._lftpFind(spec.url + artifact.getDirPath())
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
artifacts.update(newArtifacts)
return artifacts
def generate_artifact_page(ma, roots, paths, repo_url, output, groupids, optional_artifacts):
norm_repo_url = slashAtTheEnd(repo_url)
html = ("<html><head><title>Artifact {gav}</title>" + \
"<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\"></head><body>" + \
"<div class=\"header\"><a href=\"../index.html\">Back to repository summary</a></div>" + \
"<div class=\"artifact\"><h1>{gav_html}</h1>" + \
"<p class=\"breadcrumbs\"><a href=\"groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>" + \
COLON_HTML + "<a href=\"artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>" + \
COLON_HTML + "{version}</p>" + \
"<h2>Paths</h2><ul id=\"paths\">").format(gav=ma.getGAV().replace(":", " : "), gav_html=format_gav(ma.getGAV()), groupid=ma.groupId, artifactid=ma.artifactId, version=ma.version)
examples = ""
if ma.getGAV() in roots:
li = "<li>"
li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\">{aid}</a>".format(
gav=ma.getGAV().replace(":", " : "), aid=ma.artifactId, gav_filename=ma.getGAV().replace(":", "$"))
li += " <span class=\"relation\">is root</span>"
if ma.is_example():
examples += li
else:
html += li
all_paths_optional = True
directly_optional = False
for path in sorted(paths):
optional_path = False
rma = path[0].declaring
li = "<li>"
is_inherited_or_mixin = False
for rel in path:
if rel.extra and ("inherited" in rel.extra or "mixin" in rel.extra):
is_inherited_or_mixin = True
break
dec = rel.declaring
if dec:
rel_type = rel.rel_type
if dec.groupId in groupids and dec.artifactId in groupids[dec.groupId] and dec.version in groupids[dec.groupId][dec.artifactId]:
add_params = " class=\"optional\"" if optional_path else ""
li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\"{add_params}>{daid}</a>".format(
gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId,
gav_filename=dec.getGAV().replace(":", "$"), add_params=add_params)
else:
add_params = " class=\"excluded%s\"" % " optional" if optional_path else ""
li += ("<a href=\"{repo_url}{pom_path}\" title=\"{gav} (excluded, the link tries to reference the pom.xml in the same" \
+ " repo as this artifact)\"{add_params}>{daid}</a>").format(gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId,
repo_url=norm_repo_url, pom_path=dec.getPomFilepath(),
add_params=add_params)
li += " <span class=\"relation\">"
if rel_type is None:
li += "unknown relation"
elif rel_type == "DEPENDENCY":
if "embedded" in rel.extra:
if "optional" in rel.extra:
li += "embeds ?optionally?"
else:
li += "embeds"
else:
li += "depends on (scope %s" % rel.extra.split(' ', 2)[0]
if "optional" in rel.extra:
li += " optionally"
li += ")"
if "optional" in rel.extra:
optional_path = True
elif rel_type == "PARENT":
li += "has parent"
elif rel_type == "PLUGIN":
li += "uses plugin"
elif rel_type == "PLUGIN_DEP":
li += "uses plugin %s with added dependency" % rel.extra
elif rel_type == "BOM":
li += "imports BOM"
else:
li += "unknown relation (%s)" % rel_type
li += "</span> "
else:
li += "... <span class=\"relation\">unknown relation</span> "
if not is_inherited_or_mixin:
leaf = path[-1].target
gav = leaf.getGAV()
add_params = " class=\"optional\"" if optional_path else ""
li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\"{add_params}>{aid}</a></li>".format(
gav=gav.replace(":", " : "), gav_filename=gav.replace(":", "$"), aid=leaf.artifactId, add_params=add_params)
if rma.is_example():
examples += li
else:
html += li
all_paths_optional &= optional_path
directly_optional = (path[-1].rel_type == "DEPENDENCY" and "optional" in path[-1].extra)
html += examples.replace("<li>", "<li class=\"example\">")
html += "</ul></div><div id=\"pom\"><iframe src=\"{repo_url}{pom_path}\"/></div></body></html>".format(repo_url=norm_repo_url,
pom_path=ma.getPomFilepath())
with open(os.path.join(output, "pages", "artifact_version_%s.html" % ma.getGAV().replace(":", "$")), "w") as htmlfile:
htmlfile.write(html)
if len(paths) > 0 and all_paths_optional:
optional_artifacts[ma] = directly_optional
def __init__(self, indy_url):
self._indy_url = slashAtTheEnd(indy_url)
def generate_artifact_page(ma, roots, paths, repo_url, output, groupids, optional_artifacts):
norm_repo_url = slashAtTheEnd(repo_url)
html = (
"<html><head><title>Artifact {gav}</title>"
+ '<link rel="stylesheet" type="text/css" href="style.css"></head><body>'
+ '<div class="header"><a href="../index.html">Back to repository summary</a></div>'
+ '<div class="artifact"><h1>{gav_html}</h1>'
+ '<p class="breadcrumbs"><a href="groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>'
+ COLON_HTML
+ '<a href="artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>'
+ COLON_HTML
+ "{version}</p>"
+ '<h2>Paths</h2><ul id="paths">'
).format(
gav=ma.getGAV().replace(":", " : "),
gav_html=format_gav(ma.getGAV()),
groupid=ma.groupId,
artifactid=ma.artifactId,
version=ma.version,
)
examples = ""
if ma.getGAV() in roots:
li = "<li>"
li += '<a href="artifact_version_{gav_filename}.html" title="{gav}">{aid}</a>'.format(
gav=ma.getGAV().replace(":", " : "), aid=ma.artifactId, gav_filename=ma.getGAV().replace(":", "$")
)
li += ' <span class="relation">is root</span>'
if ma.is_example():
examples += li
else:
html += li
all_paths_optional = True
directly_optional = False
for path in sorted(paths):
optional_path = False
rma = path[0].declaring
li = "<li>"
is_inherited_or_mixin = False
for rel in path:
if rel.extra and ("inherited" in rel.extra or "mixin" in rel.extra):
is_inherited_or_mixin = True
break
dec = rel.declaring
if dec:
rel_type = rel.rel_type
if (
dec.groupId in groupids
and dec.artifactId in groupids[dec.groupId]
and dec.version in groupids[dec.groupId][dec.artifactId]
):
add_params = ' class="optional"' if optional_path else ""
li += '<a href="artifact_version_{gav_filename}.html" title="{gav}"{add_params}>{daid}</a>'.format(
gav=dec.getGAV().replace(":", " : "),
daid=dec.artifactId,
gav_filename=dec.getGAV().replace(":", "$"),
add_params=add_params,
)
else:
add_params = ' class="excluded%s"' % " optional" if optional_path else ""
li += (
'<a href="{repo_url}{pom_path}" title="{gav} (excluded, the link tries to reference the pom.xml in the same'
+ ' repo as this artifact)"{add_params}>{daid}</a>'
).format(
gav=dec.getGAV().replace(":", " : "),
daid=dec.artifactId,
repo_url=norm_repo_url,
pom_path=dec.getPomFilepath(),
add_params=add_params,
)
li += ' <span class="relation">'
if rel_type is None:
li += "unknown relation"
elif rel_type == "DEPENDENCY":
if "embedded" in rel.extra:
if "optional" in rel.extra:
li += "embeds ?optionally?"
else:
li += "embeds"
else:
li += "depends on (scope %s" % rel.extra.split(" ", 2)[0]
if "optional" in rel.extra:
li += " optionally"
li += ")"
if "optional" in rel.extra:
optional_path = True
elif rel_type == "PARENT":
li += "has parent"
elif rel_type == "PLUGIN":
li += "uses plugin"
elif rel_type == "PLUGIN_DEP":
li += "uses plugin %s with added dependency" % rel.extra
elif rel_type == "BOM":
li += "imports BOM"
else:
li += "unknown relation (%s)" % rel_type
li += "</span> "
else:
li += '... <span class="relation">unknown relation</span> '
if not is_inherited_or_mixin:
leaf = path[-1].target
gav = leaf.getGAV()
add_params = ' class="optional"' if optional_path else ""
li += '<a href="artifact_version_{gav_filename}.html" title="{gav}"{add_params}>{aid}</a></li>'.format(
gav=gav.replace(":", " : "),
gav_filename=gav.replace(":", "$"),
aid=leaf.artifactId,
add_params=add_params,
)
if rma.is_example():
examples += li
else:
html += li
all_paths_optional &= optional_path
directly_optional = path[-1].rel_type == "DEPENDENCY" and "optional" in path[-1].extra
html += examples.replace("<li>", '<li class="example">')
html += '</ul></div><div id="pom"><iframe src="{repo_url}{pom_path}"/></div></body></html>'.format(
repo_url=norm_repo_url, pom_path=ma.getPomFilepath()
)
with open(
os.path.join(output, "pages", "artifact_version_%s.html" % ma.getGAV().replace(":", "$")), "w"
) as htmlfile:
htmlfile.write(html)
if len(paths) > 0 and all_paths_optional:
optional_artifacts[ma] = directly_optional
def __init__(self, indy_url):
self._indy_url = slashAtTheEnd(indy_url)
def __init__(self, aprox_url):
self._aprox_url = slashAtTheEnd(aprox_url)
def _listDependencies(self, repoUrls, gavs, recursive, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(
repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s",
gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = [
'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile,
'-f', pomFilename, '-s', settingsFile
]
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning(
"Maven failed to finish with success. Skipping artifact %s",
gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn(
"Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
if artifact.artifactType in extsAndClass:
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId,
artifact.artifactId,
artifact.version, extsAndClass,
suffix, spec.url)
else:
if files:
logging.warn(
"Main artifact is missing in filelist listed from %s. Files were:\n%s",
spec.url + artifact.getDirPath(),
"\n".join(files))
else:
logging.warn(
"An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
artifacts.update(newArtifacts)
return artifacts
def __init__(self, aprox_url):
self._aprox_url = slashAtTheEnd(aprox_url)
def __init__(self, carto_url):
self._carto_url = slashAtTheEnd(carto_url)
