def _listRepository(self, repoUrls, gavPatterns): """ Loads maven artifacts from a repository. :param repoUrl: repository URL (local or remote, supported are [file://], http:// and https:// urls) :param gavPatterns: list of patterns to filter by GAV :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ prefixes = self._getPrefixes(gavPatterns) artifacts = {} for repoUrl in reversed(repoUrls): urlWithSlash = maven_repo_util.slashAtTheEnd(repoUrl) protocol = maven_repo_util.urlProtocol(urlWithSlash) if protocol == 'file': for prefix in prefixes: artifacts.update(self._listLocalRepository(urlWithSlash[7:], prefix)) elif protocol == '': for prefix in prefixes: artifacts.update(self._listLocalRepository(urlWithSlash, prefix)) elif protocol == 'http' or protocol == 'https': for prefix in prefixes: artifacts.update(self._listRemoteRepository(urlWithSlash, prefix)) else: raise "Invalid protocol!", protocol artifacts = self._filterArtifactsByPatterns(artifacts, gavPatterns) logging.debug("Found %d artifacts", len(artifacts)) return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, checksumMode, mkdirLock, filesetLock, fileset, errors): """Download artifact from a remote repository.""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = os.path.join(localRepoDir, artifact.getDirPath()) try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, True, True, filesetLock, fileset) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns): """ Loads maven artifacts from koji (brew/mead). :param kojiUrl: Koji/Brew/Mead URL :param downloadRootUrl: Download root URL of the artifacts :param tagName: Koji/Brew/Mead tag name :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL. """ kojiSession = koji.ClientSession(kojiUrl) kojiArtifacts = kojiSession.getLatestMavenArchives(tagName) gavsWithExts = {} for artifact in kojiArtifacts: # FIXME: This isn't reliable as file extension is not equal to # maven type, e.g. jar != ejb artifactType = re.search('.*\.(.+)$', artifact['filename']).group(1) gavUrl = mrbutils.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\ + artifact['build_version'] + '/' + artifact['build_release'] + '/maven/' gavu = (artifact['group_id'], artifact['artifact_id'], artifact['version'], gavUrl) gavsWithExts.setdefault(gavu, []).append(artifactType) artifacts = {} for gavu in gavsWithExts: if len(gavsWithExts[gavu]) > 1: gavsWithExts[gavu].remove("pom") for ext in gavsWithExts[gavu]: mavenArtifact = MavenArtifact(gavu[0], gavu[1], ext, gavu[2]) artifacts[mavenArtifact] = gavu[3] return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def _listRepository(self, repoUrls, gavPatterns): """ Loads maven artifacts from a repository. :param repoUrl: repository URL (local or remote, supported are [file://], http:// and https:// urls) :param gavPatterns: list of patterns to filter by GAV :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ prefixes = self._getPrefixes(gavPatterns) artifacts = {} for repoUrl in reversed(repoUrls): urlWithSlash = maven_repo_util.slashAtTheEnd(repoUrl) protocol = maven_repo_util.urlProtocol(urlWithSlash) if protocol == 'file': for prefix in prefixes: artifacts.update( self._listLocalRepository(urlWithSlash[7:], prefix)) elif protocol == '': for prefix in prefixes: artifacts.update( self._listLocalRepository(urlWithSlash, prefix)) elif protocol == 'http' or protocol == 'https': for prefix in prefixes: artifacts.update( self._listRemoteRepository(urlWithSlash, prefix)) else: raise "Invalid protocol!", protocol artifacts = self._filterArtifactsByPatterns(artifacts, gavPatterns) logging.debug("Found %d artifacts", len(artifacts)) return artifacts
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath( classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile(artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def _isArtifactInRepos(repositories, artifact): """ Returns True if specified artifact exists in at least one repositori from specified list. :param repositories: list of repository urls :param artifact: searched MavenArtifact :returns: True if specified artifact exists in at least one of the repositories. """ for repository in repositories: url = maven_repo_util.slashAtTheEnd(repository) + artifact.getDirPath() if maven_repo_util.urlExists(url): return True return False
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns): """ Loads maven artifacts from koji (brew/mead). :param kojiUrl: Koji/Brew/Mead URL :param downloadRootUrl: Download root URL of the artifacts :param tagName: Koji/Brew/Mead tag name :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ import koji kojiSession = koji.ClientSession(kojiUrl) logging.debug("Getting latest maven artifacts from tag %s.", tagName) kojiArtifacts = kojiSession.getLatestMavenArchives(tagName) filenameDict = {} for artifact in kojiArtifacts: groupId = artifact['group_id'] artifactId = artifact['artifact_id'] version = artifact['version'] gavUrl = "%s%s/%s/%s/maven/" % (maven_repo_util.slashAtTheEnd(downloadRootUrl), artifact['build_name'], artifact['build_version'], artifact['build_release']) gavu = (groupId, artifactId, version, gavUrl) filename = artifact['filename'] filenameDict.setdefault(gavu, []).append(filename) gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} } suffixes = {} # { (g,a,v,url): suffix } for gavu in filenameDict: artifactId = gavu[1] version = gavu[2] filenames = filenameDict[gavu] (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, filenames) if extsAndClass: gavuExtClass[gavu] = {} self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass) if suffix is not None: suffixes[gavu] = suffix artifacts = {} for gavu in gavuExtClass: self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3]) if gavPatterns: logging.debug("Filtering artifacts contained in the tag by GAV patterns list.") return self._filterArtifactsByPatterns(artifacts, gavPatterns, None)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns): """ Loads maven artifacts from koji (brew/mead). :param kojiUrl: Koji/Brew/Mead URL :param downloadRootUrl: Download root URL of the artifacts :param tagName: Koji/Brew/Mead tag name :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ import koji kojiSession = koji.ClientSession(kojiUrl) kojiArtifacts = kojiSession.getLatestMavenArchives(tagName) gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} } suffixes = {} # { (g,a,v,url): suffix } for artifact in kojiArtifacts: groupId = artifact['group_id'] artifactId = artifact['artifact_id'] version = artifact['version'] filename = artifact['filename'] (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifactId, version, [filename]) if extsAndClass: gavUrl = maven_repo_util.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\ + artifact['build_version'] + '/' + artifact['build_release'] + '/maven/' gavu = (groupId, artifactId, version, gavUrl) gavuExtClass.setdefault(gavu, {}) self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass) if suffix is not None and (gavu not in suffixes or suffixes[gavu] < suffix): suffixes[gavu] = suffix artifacts = {} for gavu in gavuExtClass: self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3]) return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def _listMeadTagArtifacts(self, kojiUrl, downloadRootUrl, tagName, gavPatterns): """ Loads maven artifacts from koji (brew/mead). :param kojiUrl: Koji/Brew/Mead URL :param downloadRootUrl: Download root URL of the artifacts :param tagName: Koji/Brew/Mead tag name :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL. """ import koji kojiSession = koji.ClientSession(kojiUrl) kojiArtifacts = kojiSession.getLatestMavenArchives(tagName) gavuExtClass = {} # { (g,a,v,url): {ext: set([class])} } suffixes = {} # { (g,a,v,url): suffix } for artifact in kojiArtifacts: groupId = artifact['group_id'] artifactId = artifact['artifact_id'] version = artifact['version'] filename = artifact['filename'] (extsAndClass, suffix) = self._getExtensionsAndClassifiers(artifactId, version, [filename]) if extsAndClass: gavUrl = maven_repo_util.slashAtTheEnd(downloadRootUrl) + artifact['build_name'] + '/'\ + artifact['build_version'] + '/' + artifact['build_release'] + '/maven/' gavu = (groupId, artifactId, version, gavUrl) gavuExtClass.setdefault(gavu, {}) self._updateExtensionsAndClassifiers(gavuExtClass[gavu], extsAndClass) if suffix is not None and (gavu not in suffixes or suffixes[gavu] < suffix): suffixes[gavu] = suffix artifacts = {} for gavu in gavuExtClass: self._addArtifact(artifacts, gavu[0], gavu[1], gavu[2], gavuExtClass[gavu], suffixes.get(gavu), gavu[3]) return self._filterArtifactsByPatterns(artifacts, gavPatterns)
def downloadArtifacts(remoteRepoUrl, localRepoDir, artifact, classifiers, checksumMode, mkdirLock, errors): """Download artifact from a remote repository along with pom and additional classifiers' jar""" logging.debug("Starting download of %s", str(artifact)) artifactLocalDir = localRepoDir + '/' + artifact.getDirPath() try: # handle parallelism, when two threads checks if a directory exists and then both tries to create it mkdirLock.acquire() if not os.path.exists(artifactLocalDir): os.makedirs(artifactLocalDir) mkdirLock.release() remoteRepoUrl = maven_repo_util.slashAtTheEnd(remoteRepoUrl) # Download main artifact artifactUrl = remoteRepoUrl + artifact.getArtifactFilepath() artifactLocalPath = os.path.join(localRepoDir, artifact.getArtifactFilepath()) maven_repo_util.fetchFile(artifactUrl, artifactLocalPath, checksumMode, exitOnError=True) if not artifact.getClassifier(): # Download pom if the main type is not pom if artifact.getArtifactFilename() != artifact.getPomFilename(): artifactPomUrl = remoteRepoUrl + artifact.getPomFilepath() artifactPomLocalPath = os.path.join(localRepoDir, artifact.getPomFilepath()) maven_repo_util.fetchFile(artifactPomUrl, artifactPomLocalPath, checksumMode, exitOnError=True) # Download additional classifiers (only for non-pom artifacts) for classifier in classifiers: artifactClassifierUrl = remoteRepoUrl + artifact.getClassifierFilepath(classifier) if maven_repo_util.urlExists(artifactClassifierUrl): artifactClassifierLocalPath = os.path.join( localRepoDir, artifact.getClassifierFilepath(classifier)) maven_repo_util.fetchFile( artifactClassifierUrl, artifactClassifierLocalPath, checksumMode, exitOnError=True) except BaseException as ex: logging.error("Error while downloading artifact %s: %s", artifact, str(ex)) errors.put(ex)
def __init__(self, carto_url): self._carto_url = slashAtTheEnd(carto_url)
def _listDependencies(self, repoUrls, gavs, recursive, include_scope, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :param recursive: runs dependency:list recursively using the previously discovered dependencies if True :param include_scope: defines scope which will be used when running mvn as includeScope parameter, can be None to use Maven's default :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] if include_scope: args.append("-DincludeScope=%s" % include_scope) logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.isAllClassifiers(): resultingArtifacts = {} for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn("Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if artifact.artifactType in extsAndClass: self._addArtifact(resultingArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn("Main artifact (%s) is missing in filelist listed from %s. Files were:\n%s", artifact.artifactType, spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn("An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) newArtifacts = resultingArtifacts artifacts.update(newArtifacts) return artifacts
def _listDependencies(self, repoUrls, gavs): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is it's repo root URL, or empty dictionary if something goes wrong. """ artifacts = {} for gav in gavs: logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd(repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" with open(settingsFile, 'w') as settings: settingsContent = re.sub('\$url', successPomUrl, self.SETTINGS_TPL) settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = ['mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile] logging.debug("Running Maven:\n %s", " ".join(args)) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning("Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) newArtifacts = self._listArtifacts(repoUrls, gavList) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] out = self._lftpFind(spec.url + artifact.getDirPath()) files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) artifacts.update(newArtifacts) return artifacts
def generate_artifact_page(ma, roots, paths, repo_url, output, groupids, optional_artifacts): norm_repo_url = slashAtTheEnd(repo_url) html = ("<html><head><title>Artifact {gav}</title>" + \ "<link rel=\"stylesheet\" type=\"text/css\" href=\"style.css\"></head><body>" + \ "<div class=\"header\"><a href=\"../index.html\">Back to repository summary</a></div>" + \ "<div class=\"artifact\"><h1>{gav_html}</h1>" + \ "<p class=\"breadcrumbs\"><a href=\"groupid_{groupid}.html\" title=\"GroupId {groupid}\">{groupid}</a>" + \ COLON_HTML + "<a href=\"artifactid_{groupid}${artifactid}.html\" title=\"ArtifactId {artifactid}\">{artifactid}</a>" + \ COLON_HTML + "{version}</p>" + \ "<h2>Paths</h2><ul id=\"paths\">").format(gav=ma.getGAV().replace(":", " : "), gav_html=format_gav(ma.getGAV()), groupid=ma.groupId, artifactid=ma.artifactId, version=ma.version) examples = "" if ma.getGAV() in roots: li = "<li>" li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\">{aid}</a>".format( gav=ma.getGAV().replace(":", " : "), aid=ma.artifactId, gav_filename=ma.getGAV().replace(":", "$")) li += " <span class=\"relation\">is root</span>" if ma.is_example(): examples += li else: html += li all_paths_optional = True directly_optional = False for path in sorted(paths): optional_path = False rma = path[0].declaring li = "<li>" is_inherited_or_mixin = False for rel in path: if rel.extra and ("inherited" in rel.extra or "mixin" in rel.extra): is_inherited_or_mixin = True break dec = rel.declaring if dec: rel_type = rel.rel_type if dec.groupId in groupids and dec.artifactId in groupids[dec.groupId] and dec.version in groupids[dec.groupId][dec.artifactId]: add_params = " class=\"optional\"" if optional_path else "" li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\"{add_params}>{daid}</a>".format( gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId, gav_filename=dec.getGAV().replace(":", "$"), add_params=add_params) else: add_params = " class=\"excluded%s\"" % " optional" if optional_path else "" li += ("<a href=\"{repo_url}{pom_path}\" title=\"{gav} (excluded, the link tries to reference the pom.xml in the same" \ + " repo as this artifact)\"{add_params}>{daid}</a>").format(gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId, repo_url=norm_repo_url, pom_path=dec.getPomFilepath(), add_params=add_params) li += " <span class=\"relation\">" if rel_type is None: li += "unknown relation" elif rel_type == "DEPENDENCY": if "embedded" in rel.extra: if "optional" in rel.extra: li += "embeds ?optionally?" else: li += "embeds" else: li += "depends on (scope %s" % rel.extra.split(' ', 2)[0] if "optional" in rel.extra: li += " optionally" li += ")" if "optional" in rel.extra: optional_path = True elif rel_type == "PARENT": li += "has parent" elif rel_type == "PLUGIN": li += "uses plugin" elif rel_type == "PLUGIN_DEP": li += "uses plugin %s with added dependency" % rel.extra elif rel_type == "BOM": li += "imports BOM" else: li += "unknown relation (%s)" % rel_type li += "</span> " else: li += "... <span class=\"relation\">unknown relation</span> " if not is_inherited_or_mixin: leaf = path[-1].target gav = leaf.getGAV() add_params = " class=\"optional\"" if optional_path else "" li += "<a href=\"artifact_version_{gav_filename}.html\" title=\"{gav}\"{add_params}>{aid}</a></li>".format( gav=gav.replace(":", " : "), gav_filename=gav.replace(":", "$"), aid=leaf.artifactId, add_params=add_params) if rma.is_example(): examples += li else: html += li all_paths_optional &= optional_path directly_optional = (path[-1].rel_type == "DEPENDENCY" and "optional" in path[-1].extra) html += examples.replace("<li>", "<li class=\"example\">") html += "</ul></div><div id=\"pom\"><iframe src=\"{repo_url}{pom_path}\"/></div></body></html>".format(repo_url=norm_repo_url, pom_path=ma.getPomFilepath()) with open(os.path.join(output, "pages", "artifact_version_%s.html" % ma.getGAV().replace(":", "$")), "w") as htmlfile: htmlfile.write(html) if len(paths) > 0 and all_paths_optional: optional_artifacts[ma] = directly_optional
def __init__(self, indy_url): self._indy_url = slashAtTheEnd(indy_url)
def generate_artifact_page(ma, roots, paths, repo_url, output, groupids, optional_artifacts): norm_repo_url = slashAtTheEnd(repo_url) html = ( "<html><head><title>Artifact {gav}</title>" + '<link rel="stylesheet" type="text/css" href="style.css"></head><body>' + '<div class="header"><a href="../index.html">Back to repository summary</a></div>' + '<div class="artifact"><h1>{gav_html}</h1>' + '<p class="breadcrumbs"><a href="groupid_{groupid}.html" title="GroupId {groupid}">{groupid}</a>' + COLON_HTML + '<a href="artifactid_{groupid}${artifactid}.html" title="ArtifactId {artifactid}">{artifactid}</a>' + COLON_HTML + "{version}</p>" + '<h2>Paths</h2><ul id="paths">' ).format( gav=ma.getGAV().replace(":", " : "), gav_html=format_gav(ma.getGAV()), groupid=ma.groupId, artifactid=ma.artifactId, version=ma.version, ) examples = "" if ma.getGAV() in roots: li = "<li>" li += '<a href="artifact_version_{gav_filename}.html" title="{gav}">{aid}</a>'.format( gav=ma.getGAV().replace(":", " : "), aid=ma.artifactId, gav_filename=ma.getGAV().replace(":", "$") ) li += ' <span class="relation">is root</span>' if ma.is_example(): examples += li else: html += li all_paths_optional = True directly_optional = False for path in sorted(paths): optional_path = False rma = path[0].declaring li = "<li>" is_inherited_or_mixin = False for rel in path: if rel.extra and ("inherited" in rel.extra or "mixin" in rel.extra): is_inherited_or_mixin = True break dec = rel.declaring if dec: rel_type = rel.rel_type if ( dec.groupId in groupids and dec.artifactId in groupids[dec.groupId] and dec.version in groupids[dec.groupId][dec.artifactId] ): add_params = ' class="optional"' if optional_path else "" li += '<a href="artifact_version_{gav_filename}.html" title="{gav}"{add_params}>{daid}</a>'.format( gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId, gav_filename=dec.getGAV().replace(":", "$"), add_params=add_params, ) else: add_params = ' class="excluded%s"' % " optional" if optional_path else "" li += ( '<a href="{repo_url}{pom_path}" title="{gav} (excluded, the link tries to reference the pom.xml in the same' + ' repo as this artifact)"{add_params}>{daid}</a>' ).format( gav=dec.getGAV().replace(":", " : "), daid=dec.artifactId, repo_url=norm_repo_url, pom_path=dec.getPomFilepath(), add_params=add_params, ) li += ' <span class="relation">' if rel_type is None: li += "unknown relation" elif rel_type == "DEPENDENCY": if "embedded" in rel.extra: if "optional" in rel.extra: li += "embeds ?optionally?" else: li += "embeds" else: li += "depends on (scope %s" % rel.extra.split(" ", 2)[0] if "optional" in rel.extra: li += " optionally" li += ")" if "optional" in rel.extra: optional_path = True elif rel_type == "PARENT": li += "has parent" elif rel_type == "PLUGIN": li += "uses plugin" elif rel_type == "PLUGIN_DEP": li += "uses plugin %s with added dependency" % rel.extra elif rel_type == "BOM": li += "imports BOM" else: li += "unknown relation (%s)" % rel_type li += "</span> " else: li += '... <span class="relation">unknown relation</span> ' if not is_inherited_or_mixin: leaf = path[-1].target gav = leaf.getGAV() add_params = ' class="optional"' if optional_path else "" li += '<a href="artifact_version_{gav_filename}.html" title="{gav}"{add_params}>{aid}</a></li>'.format( gav=gav.replace(":", " : "), gav_filename=gav.replace(":", "$"), aid=leaf.artifactId, add_params=add_params, ) if rma.is_example(): examples += li else: html += li all_paths_optional &= optional_path directly_optional = path[-1].rel_type == "DEPENDENCY" and "optional" in path[-1].extra html += examples.replace("<li>", '<li class="example">') html += '</ul></div><div id="pom"><iframe src="{repo_url}{pom_path}"/></div></body></html>'.format( repo_url=norm_repo_url, pom_path=ma.getPomFilepath() ) with open( os.path.join(output, "pages", "artifact_version_%s.html" % ma.getGAV().replace(":", "$")), "w" ) as htmlfile: htmlfile.write(html) if len(paths) > 0 and all_paths_optional: optional_artifacts[ma] = directly_optional
def __init__(self, aprox_url): self._aprox_url = slashAtTheEnd(aprox_url)
def _listDependencies(self, repoUrls, gavs, recursive, skipmissing): """ Loads maven artifacts from mvn dependency:list. :param repoUrls: URL of the repositories that contains the listed artifacts :param gavs: List of top level GAVs :returns: Dictionary where index is MavenArtifact object and value is ArtifactSpec with its repo root URL """ artifacts = {} workingSet = set(gavs) checkedSet = set() while workingSet: gav = workingSet.pop() checkedSet.add(gav) logging.debug("Resolving dependencies for %s", gav) artifact = MavenArtifact.createFromGAV(gav) pomFilename = 'poms/' + artifact.getPomFilename() successPomUrl = None fetched = False for repoUrl in repoUrls: pomUrl = maven_repo_util.slashAtTheEnd( repoUrl) + artifact.getPomFilepath() fetched = maven_repo_util.fetchFile(pomUrl, pomFilename) if fetched: successPomUrl = repoUrl break if not fetched: logging.warning("Failed to retrieve pom file for artifact %s", gav) continue tempDir = maven_repo_util.getTempDir() if not os.path.exists(tempDir): os.makedirs(tempDir) # Create settings.xml settingsFile = tempDir + "settings.xml" settingsContent = self.SETTINGS_TPL.replace('${url}', successPomUrl) \ .replace('${temp}', maven_repo_util.getTempDir()) with open(settingsFile, 'w') as settings: settings.write(settingsContent) # Build dependency:list depsDir = tempDir + "maven-deps-output/" outFile = depsDir + gav + ".out" args = [ 'mvn', 'dependency:list', '-N', '-DoutputFile=' + outFile, '-f', pomFilename, '-s', settingsFile ] logging.debug("Running Maven:\n %s", " ".join(args)) logging.debug("settings.xml contents: %s", settingsContent) mvn = Popen(args, stdout=PIPE) mvnStdout = mvn.communicate()[0] logging.debug("Maven output:\n%s", mvnStdout) if mvn.returncode != 0: logging.warning( "Maven failed to finish with success. Skipping artifact %s", gav) continue with open(outFile, 'r') as out: depLines = out.readlines() gavList = self._parseDepList(depLines) logging.debug("Resolved dependencies of %s: %s", gav, str(gavList)) newArtifacts = self._listArtifacts(repoUrls, gavList) if recursive: for artifact in newArtifacts: ngav = artifact.getGAV() if ngav not in checkedSet: workingSet.add(ngav) if self.configuration.allClassifiers: for artifact in newArtifacts.keys(): spec = newArtifacts[artifact] try: out = self._lftpFind(spec.url + artifact.getDirPath()) except IOError as ex: if skipmissing: logging.warn( "Error while listing files in %s: %s. Skipping...", spec.url + artifact.getDirPath(), str(ex)) continue else: raise ex files = [] for line in out.split('\n'): if line != "./" and line != "": files.append(line[2:]) (extsAndClass, suffix) = self._getExtensionsAndClassifiers( artifact.artifactId, artifact.version, files) if len(extsAndClass) > 1 and "pom" in extsAndClass: del extsAndClass["pom"] if artifact.artifactType in extsAndClass: spec.classifiers = extsAndClass[artifact.artifactType] del extsAndClass[artifact.artifactType] self._addArtifact(newArtifacts, artifact.groupId, artifact.artifactId, artifact.version, extsAndClass, suffix, spec.url) else: if files: logging.warn( "Main artifact is missing in filelist listed from %s. Files were:\n%s", spec.url + artifact.getDirPath(), "\n".join(files)) else: logging.warn( "An empty filelist was listed from %s. Skipping...", spec.url + artifact.getDirPath()) artifacts.update(newArtifacts) return artifacts