def copyArtifact(remoteRepoPath, localRepoDir, artifact, checksumMode): """Copy artifact from a repository on the local file system along with pom and source jar""" # Copy main artifact artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath()) artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath): maven_repo_util.fetchFile(artifactPath, artifactLocalPath, checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors): """Download artifact from a remote repository.""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath()) try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, True, True, filesetLock, fileset) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def copyArtifact(remoteRepoPath, localRepoDir, artifact, classifiers, checksumMode): """Copy artifact from a repository on the local file system along with pom and source jar""" # Copy main artifact artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath()) artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath): maven_repo_util.fetchFile(artifactPath, artifactLocalPath, checksumMode) if not artifact.getClassifier(): # Copy pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomPath = os.path.join(remoteRepoPath, artifact.getPomFilepath()) artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) if os.path.exists(artifactPomPath ) and not os.path.exists(artifactPomLocalPath): maven_repo_util.fetchFile(artifactPomPath, artifactPomLocalPath, checksumMode) # Copy additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierPath = os.path.join( remoteRepoPath, artifact.getClassifierFilepath(classifier)) artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) if os.path.exists( artifactClassifierPath ) and not os.path.exists(artifactClassifierLocalPath): maven_repo_util.fetchFile(artifactClassifierPath, artifactClassifierLocalPath, checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath( classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile(artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def copyArtifact(remoteRepoPath, localRepoDir, artifact, classifiers, checksumMode): """Copy artifact from a repository on the local file system along with pom and source jar""" # Copy main artifact artifactPath = os.path.join(remoteRepoPath, artifact.getArtifactFilepath()) artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) if os.path.exists(artifactPath) and not os.path.exists(artifactLocalPath): maven_repo_util.fetchFile(artifactPath, artifactLocalPath, checksumMode) if not artifact.getClassifier(): # Copy pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomPath = os.path.join(remoteRepoPath, artifact.getPomFilepath()) artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) if os.path.exists(artifactPomPath) and not os.path.exists(artifactPomLocalPath): maven_repo_util.fetchFile(artifactPomPath, artifactPomLocalPath, checksumMode) # Copy additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierPath = os.path.join(remoteRepoPath, artifact.getClassifierFilepath(classifier)) artifactClassifierLocalPath = os.path.join(localRepoDir, artifact.getClassifierFilepath(classifier)) if os.path.exists(artifactClassifierPath) and not os.path.exists(artifactClassifierLocalPath): maven_repo_util.fetchFile(artifactClassifierPath, artifactClassifierLocalPath, checksumMode)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile( artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :param recursive: runs dependency:list recursively using the previously discovered dependencies if True :param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None to use Maven's default :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] if include_scope: args.append("-DincludeScope=%s" % include_scope) logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.isAllClassifiers(): resultingArtifacts = {} for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn("Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if artifact.artifactType in extsAndClass: self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s", artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn("An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) newArtifacts = resultingArtifacts artifacts.update(newArtifacts) return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd( repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = [ 'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile ] logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning( "Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn( "Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] if artifact.artifactType in extsAndClass: spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn( "Main artifact is missing in filelist listed from %s. Files were:\n%s", spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn( "An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) artifacts.update(newArtifacts) return artifacts
def _listDependencies(self, repoUrls, gavs): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL, or empty dictionary if something goes wrong. """ artifacts = {} for gav in gavs: logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" with open(settingsFile, 'w') as settings: settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL) settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] logging.debug("Running Maven:\n %s", " ".join(args)) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) newArtifacts = self._listArtifacts(repoUrls, gavList) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] out = self._lftpFind(spec.url + artifact.getDirPath()) files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) artifacts.update(newArtifacts) return artifacts