def _lftpFind(self, url):
if maven_repo_util.urlExists(url):
lftp = Popen(r'lftp -c "set ssl:verify-certificate no ; open ' + url
+ ' && find ."', stdout=PIPE, shell=True)
result = lftp.communicate()[0]
if lftp.returncode:
raise IOError("lftp find in %s ended by return code %d" % (url, lftp.returncode))
else:
return result
else:
raise IOError("Cannot list URL %s. The URL does not exist." % url)
def _lftpFind(self, url):
if maven_repo_util.urlExists(url):
lftp = Popen(r'lftp -c "set ssl:verify-certificate no ; open ' + url
+ ' && find ."', stdout=PIPE, shell=True)
result = lftp.communicate()[0]
if lftp.returncode:
raise IOError("lftp find in %s ended by return code %d" % (url, lftp.returncode))
else:
return result
else:
raise IOError("Cannot list URL %s. The URL does not exist." % url)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers,
checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir,
artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl,
artifactLocalPath,
checksumMode,
exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir,
artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl,
artifactPomLocalPath,
checksumMode,
exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(
classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir,
artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(artifactClassifierUrl,
artifactClassifierLocalPath,
checksumMode,
exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact,
str(ex))
errors.put(ex)
def _isArtifactInRepos(repositories, artifact):
"""
Returns True if specified artifact exists in at least one repositori from specified list.
:param repositories: list of repository urls
:param artifact: searched MavenArtifact
:returns: True if specified artifact exists in at least one of the repositories.
"""
for repository in repositories:
url = maven_repo_util.slashAtTheEnd(repository) + artifact.getDirPath()
if maven_repo_util.urlExists(url):
return True
return False
def _listArtifacts(self, urls, gavs):
"""
Loads maven artifacts from list of GAVs and tries to locate the artifacts in one of the
specified repositories.
:param urls: repository URLs where the given GAVs can be located
:param gavs: List of GAVs
:returns: Dictionary where index is MavenArtifact object and value is it's repo root URL.
"""
artifacts = {}
for gav in gavs:
artifact = MavenArtifact.createFromGAV(gav)
for url in urls:
gavUrl = url + '/' + artifact.getDirPath()
if mrbutils.urlExists(gavUrl):
artifacts[artifact] = url
break
if not artifact in artifacts:
logging.warning('artifact %s not found in any url!', artifact)
return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors):
"""Download artifact from a remote repository along with pom and additional classifiers' jar"""
logging.debug("Starting download of %s", str(artifact))
artifactLocalDir = localRepoDir + '/' + artifact.getDirPath()
try:
# handle parallelism, when two threads checks if a directory exists and then both tries to create it
mkdirLock.acquire()
if not os.path.exists(artifactLocalDir):
os.makedirs(artifactLocalDir)
mkdirLock.release()
remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl)
# Download main artifact
artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath()
artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath())
maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True)
if not artifact.getClassifier():
# Download pom if the main type is not pom
if artifact.getArtifactFilename() != artifact.getPomFilename():
artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath()
artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath())
maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True)
# Download additional classifiers (only for non-pom artifacts)
for classifier in classifiers:
artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier)
if maven_repo_util.urlExists(artifactClassifierUrl):
artifactClassifierLocalPath = os.path.join(
localRepoDir, artifact.getClassifierFilepath(classifier))
maven_repo_util.fetchFile(
artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True)
except BaseException as ex:
logging.error("Error while downloading artifact %s: %s", artifact, str(ex))
errors.put(ex)
