def _listDependencies(self, repoUrls, gavs, recursive, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd( repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = [ 'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile ] logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning( "Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn( "Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] if artifact.artifactType in extsAndClass: spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn( "Main artifact is missing in filelist listed from %s. Files were:\n%s", spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn( "An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) artifacts.update(newArtifacts) return artifacts
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :param recursive: runs dependency:list recursively using the previously discovered dependencies if True :param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None to use Maven's default :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] if include_scope: args.append("-DincludeScope=%s" % include_scope) logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.isAllClassifiers(): resultingArtifacts = {} for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn("Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if artifact.artifactType in extsAndClass: self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s", artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn("An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) newArtifacts = resultingArtifacts artifacts.update(newArtifacts) return artifacts
def _listDependencies(self, repoUrls, gavs): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL, or empty dictionary if something goes wrong. """ artifacts = {} for gav in gavs: logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" with open(settingsFile, 'w') as settings: settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL) settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] logging.debug("Running Maven:\n %s", " ".join(args)) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) newArtifacts = self._listArtifacts(repoUrls, gavList) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] out = self._lftpFind(spec.url + artifact.getDirPath()) files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) artifacts.update(newArtifacts) return artifacts