def _lftpFind(self, url): if maven_repo_util.urlExists(url): lftp = Popen(r'lftp -c "set ssl:verify-certificate no ; open ' + url + ' && find ."', stdout=PIPE, shell=True) result = lftp.communicate()[0] if lftp.returncode: raise IOError("lftp find in %s ended by return code %d" % (url, lftp.returncode)) else: return result else: raise IOError("Cannot list URL %s. The URL does not exist." % url)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath( classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile(artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def _isArtifactInRepos(repositories, artifact): """ Returns True if specified artifact exists in at least one repositori from specified list. :param repositories: list of repository urls :param artifact: searched MavenArtifact :returns: True if specified artifact exists in at least one of the repositories. """ for repository in repositories: url = maven_repo_util.slashAtTheEnd(repository) + artifact.getDirPath() if maven_repo_util.urlExists(url): return True return False
def _listArtifacts(self, urls, gavs): """ Loads maven artifacts from list of GAVs and tries to locate the artifacts in one of the specified repositories. :param urls: repository URLs where the given GAVs can be located :param gavs: List of GAVs :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL. """ artifacts = {} for gav in gavs: artifact = MavenArtifact.createFromGAV(gav) for url in urls: gavUrl = url + '/' + artifact.getDirPath() if mrbutils.urlExists(gavUrl): artifacts[artifact] = url break if not artifact in artifacts: logging.warning('artifact %s not found in any url!', artifact) return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile( artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)