def copyArtifact(remoteRepoPath, localRepoDir, artifact, checksumMode):
"""Copy artifact from a repository on the local file system along with pom and source jar"""
# Copy main artifact
artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath())
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath):
maven_repo_util.fetchFile(artifactPath, artifactLocalPath, checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode,
mkdirLock, filesetLock, fileset, errors):
"""Download artifact from a remote repository."""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath())
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode,
True, True, filesetLock, fileset)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact,
str(ex))
errors.put(ex)
def copyArtifact(remoteRepoPath, localRepoDir, artifact, classifiers,
checksumMode):
"""Copy artifact from a repository on the local file system along with pom and source jar"""
# Copy main artifact
artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath())
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath):
maven_repo_util.fetchFile(artifactPath, artifactLocalPath,
checksumMode)
if not artifact.getClassifier():
# Copy pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomPath = os.path.join(remoteRepoPath,
artifact.getPomFilepath())
artifactPomLocalPath = os.path.join(localRepoDir,
artifact.getPomFilepath())
if os.path.exists(artifactPomPath
) and not os.path.exists(artifactPomLocalPath):
maven_repo_util.fetchFile(artifactPomPath,
artifactPomLocalPath, checksumMode)
# Copy additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierPath = os.path.join(
remoteRepoPath, artifact.getClassifierFilepath(classifier))
artifactClassifierLocalPath = os.path.join(
localRepoDir, artifact.getClassifierFilepath(classifier))
if os.path.exists(
artifactClassifierPath
) and not os.path.exists(artifactClassifierLocalPath):
maven_repo_util.fetchFile(artifactClassifierPath,
artifactClassifierLocalPath,
checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors):
"""Download artifact from a remote repository."""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath())
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, True, True, filesetLock, fileset)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact, str(ex))
errors.put(ex)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers,
checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl,
artifactLocalPath,
checksumMode,
exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir,
artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl,
artifactPomLocalPath,
checksumMode,
exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(
classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir,
artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(artifactClassifierUrl,
artifactClassifierLocalPath,
checksumMode,
exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact,
str(ex))
errors.put(ex)
def copyArtifact(remoteRepoPath, localRepoDir, artifact, classifiers, checksumMode):
"""Copy artifact from a repository on the local file system along with pom and source jar"""
# Copy main artifact
artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath())
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath):
maven_repo_util.fetchFile(artifactPath, artifactLocalPath, checksumMode)
if not artifact.getClassifier():
# Copy pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomPath = os.path.join(remoteRepoPath, artifact.getPomFilepath())
artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath())
if os.path.exists(artifactPomPath) and not os.path.exists(artifactPomLocalPath):
maven_repo_util.fetchFile(artifactPomPath, artifactPomLocalPath, checksumMode)
# Copy additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierPath = os.path.join(remoteRepoPath, artifact.getClassifierFilepath(classifier))
artifactClassifierLocalPath = os.path.join(localRepoDir, artifact.getClassifierFilepath(classifier))
if os.path.exists(artifactClassifierPath) and not os.path.exists(artifactClassifierLocalPath):
maven_repo_util.fetchFile(artifactClassifierPath, artifactClassifierLocalPath, checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir, artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(
artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact, str(ex))
errors.put(ex)
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:param recursive: runs dependency:list recursively using the previously discovered dependencies if True
:param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None
to use Maven's default
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
if include_scope:
args.append("-DincludeScope=%s" % include_scope)
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.isAllClassifiers():
resultingArtifacts = {}
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn("Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if artifact.artifactType in extsAndClass:
self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
else:
if files:
logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s",
artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files))
else:
logging.warn("An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
newArtifacts = resultingArtifacts
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, skipmissing):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
ArtifactSpec with its repo root URL
"""
artifacts = {}
workingSet = set(gavs)
checkedSet = set()
while workingSet:
gav = workingSet.pop()
checkedSet.add(gav)
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(
repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s",
gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \
.replace('${temp}', maven_repo_util.getTempDir())
with open(settingsFile, 'w') as settings:
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = [
'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile,
'-f', pomFilename, '-s', settingsFile
]
logging.debug("Running Maven:\n %s", " ".join(args))
logging.debug("settings.xml contents: %s", settingsContent)
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning(
"Maven failed to finish with success. Skipping artifact %s",
gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
logging.debug("Resolved dependencies of %s: %s", gav, str(gavList))
newArtifacts = self._listArtifacts(repoUrls, gavList)
if recursive:
for artifact in newArtifacts:
ngav = artifact.getGAV()
if ngav not in checkedSet:
workingSet.add(ngav)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
try:
out = self._lftpFind(spec.url + artifact.getDirPath())
except IOError as ex:
if skipmissing:
logging.warn(
"Error while listing files in %s: %s. Skipping...",
spec.url + artifact.getDirPath(), str(ex))
continue
else:
raise ex
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
if artifact.artifactType in extsAndClass:
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId,
artifact.artifactId,
artifact.version, extsAndClass,
suffix, spec.url)
else:
if files:
logging.warn(
"Main artifact is missing in filelist listed from %s. Files were:\n%s",
spec.url + artifact.getDirPath(),
"\n".join(files))
else:
logging.warn(
"An empty filelist was listed from %s. Skipping...",
spec.url + artifact.getDirPath())
artifacts.update(newArtifacts)
return artifacts
def _listDependencies(self, repoUrls, gavs):
"""
Loads maven artifacts from mvn dependency:list.
:param repoUrls: URL of the repositories that contains the listed artifacts
:param gavs: List of top level GAVs
:returns: Dictionary where index is MavenArtifact object and value is
it's repo root URL, or empty dictionary if something goes wrong.
"""
artifacts = {}
for gav in gavs:
logging.debug("Resolving dependencies for %s", gav)
artifact = MavenArtifact.createFromGAV(gav)
pomFilename = 'poms/' + artifact.getPomFilename()
successPomUrl = None
fetched = False
for repoUrl in repoUrls:
pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath()
fetched = maven_repo_util.fetchFile(pomUrl, pomFilename)
if fetched:
successPomUrl = repoUrl
break
if not fetched:
logging.warning("Failed to retrieve pom file for artifact %s", gav)
continue
tempDir = maven_repo_util.getTempDir()
if not os.path.exists(tempDir):
os.makedirs(tempDir)
# Create settings.xml
settingsFile = tempDir + "settings.xml"
with open(settingsFile, 'w') as settings:
settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL)
settings.write(settingsContent)
# Build dependency:list
depsDir = tempDir + "maven-deps-output/"
outFile = depsDir + gav + ".out"
args = ['mvn', 'dependency:list', '-N',
'-DoutputFile=' + outFile,
'-f', pomFilename,
'-s', settingsFile]
logging.debug("Running Maven:\n %s", " ".join(args))
mvn = Popen(args, stdout=PIPE)
mvnStdout = mvn.communicate()[0]
logging.debug("Maven output:\n%s", mvnStdout)
if mvn.returncode != 0:
logging.warning("Maven failed to finish with success. Skipping artifact %s", gav)
continue
with open(outFile, 'r') as out:
depLines = out.readlines()
gavList = self._parseDepList(depLines)
newArtifacts = self._listArtifacts(repoUrls, gavList)
if self.configuration.allClassifiers:
for artifact in newArtifacts.keys():
spec = newArtifacts[artifact]
out = self._lftpFind(spec.url + artifact.getDirPath())
files = []
for line in out.split('\n'):
if line != "./" and line != "":
files.append(line[2:])
(extsAndClass, suffix) = self._getExtensionsAndClassifiers(
artifact.artifactId, artifact.version, files)
if len(extsAndClass) > 1 and "pom" in extsAndClass:
del extsAndClass["pom"]
spec.classifiers = extsAndClass[artifact.artifactType]
del extsAndClass[artifact.artifactType]
self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId,
artifact.version, extsAndClass, suffix, spec.url)
artifacts.update(newArtifacts)
return artifacts
