def _listDependencies(self, repoUrls, gavs, recursive, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(
repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s",
gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = [
'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile,
'-f', pomFilename, '-s', settingsFile
]
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning(
"Maven failed to finish with success. Skipping artifact %s",
gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn(
"Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
if artifact.artifactType in extsAndClass:
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId,
artifact.artifactId,
artifact.version, extsAndClass,
suffix, spec.url)
else:
if files:
logging.warn(
"Main artifact is missing in filelist listed from %s. Files were:\n%s",
spec.url + artifact.getDirPath(),
"\n".join(files))
else:
logging.warn(
"An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:param recursive: runs dependency:list recursively using the previously discovered dependencies if True
:param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None
to use Maven's default
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
if include_scope:
args.append("-DincludeScope=%s" % include_scope)
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.isAllClassifiers():
resultingArtifacts = {}
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn("Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if artifact.artifactType in extsAndClass:
self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
else:
if files:
logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s",
artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files))
else:
logging.warn("An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
newArtifacts = resultingArtifacts
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
it's repo root URL, or empty dictionary if something goes wrong.
"""
artifacts = {}
for gav in gavs:
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
with open(settingsFile, 'w') as settings:
settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL)
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
logging.debug("Running Maven:\n %s", " ".join(args))
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
newArtifacts = self._listArtifacts(repoUrls, gavList)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
out = self._lftpFind(spec.url + artifact.getDirPath())
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
artifacts.update(newArtifacts)
return artifacts
