def _filterMultipleVersions(self, artifactList): logging.debug("Filtering multi-version artifacts to have just a single version.") regExps = maven_repo_util.getRegExpsFromStrings(self.config.multiVersionGAs, False) for ga in sorted(artifactList.keys()): if maven_repo_util.somethingMatch(regExps, ga): continue # Gather all priorities priorities = sorted(artifactList[ga].keys()) priority = priorities[0] # Gather all versions versions = list(artifactList[ga][priority].keys()) if len(versions) > 1: # list of 1 is sorted by definition versions = maven_repo_util._sortVersionsWithAtlas(versions) # Remove version, priorities and gats from artifactList as necessary for version in versions[1:]: logging.debug("Dropping GAV %s:%s from priority %i because only single version is allowed.", ga, version, priority) del artifactList[ga][priority][version] for p in priorities[1:]: logging.debug("Dropping GA %s from priority %i because of no version left.", ga, p) del artifactList[ga][p] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList
def _filterExcludedTypes(self, artifactList): ''' Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with not-excluded artifact types only. :param artifactList: artifactList to be filtered. :param exclTypes: list of excluded types :returns: artifactList without artifacts that matched specified types and had no other main types. ''' logging.debug("Filtering artifacts with excluded types.") regExps = maven_repo_util.getRegExpsFromStrings(self.config.gatcvWhitelist) exclTypes = self.config.excludedTypes for ga in artifactList.keys(): for priority in artifactList[ga].keys(): for version in artifactList[ga][priority].keys(): artSpec = artifactList[ga][priority][version] for artType in list(artSpec.artTypes.keys()): if artType in exclTypes: artTypeObj = artSpec.artTypes[artType] classifiers = artTypeObj.classifiers (groupId, artifactId) = ga.split(':') for classifier in list(classifiers): art = MavenArtifact(groupId, artifactId, artType, version, classifier) gatcv = art.getGATCV() if not maven_repo_util.somethingMatch(regExps, gatcv): logging.debug("Dropping classifier \"%s\" of %s:%s:%s from priority %i because of " "excluded type.", classifier, ga, artType, version, priority) classifiers.remove(classifier) else: logging.debug("Skipping drop of %s:%s:%s:%s from priority %i because it matches a " "whitelist pattern.", ga, artType, classifier, version, priority) if not classifiers: logging.debug("Dropping %s:%s:%s from priority %i because of no classifier left.", ga, artType, version, priority) del(artSpec.artTypes[artType]) noMain = True for artType in artSpec.artTypes.keys(): artTypeObj = artSpec.artTypes[artType] if artTypeObj.mainType: noMain = False break if not artSpec.artTypes or noMain: if noMain: logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.", ga, version, priority) else: logging.debug("Dropping GAV %s:%s from priority %i because of no artifact type left.", ga, version, priority) del artifactList[ga][priority][version] if not artifactList[ga][priority]: logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority) del artifactList[ga][priority] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList
def _filterArtifactsByPatterns(self, artifacts, gavPatterns): if not gavPatterns: return artifacts regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns) includedArtifacts = {} for artifact in artifacts: if maven_repo_util.somethingMatch(regExps, artifact.getGAV()): includedArtifacts[artifact] = artifacts[artifact] return includedArtifacts
def _filterExcludedGAVs(self, artifactList): """ Filter artifactList removing specified GAVs. :param artifactList: artifactList to be filtered. :returns: artifactList without artifacts that matched specified GAVs. """ logging.debug("Filtering artifacts with excluded GAVs.") regExps = maven_repo_util.getRegExpsFromStrings(self.config.excludedGAVs) gavRegExps = [] gatcvRegExps = [] for regExp in regExps: if regExp.pattern.count(":") > 2: gatcvRegExps.append(regExp) else: gavRegExps.append(regExp) for ga in artifactList.keys(): for priority in artifactList[ga].keys(): for version in artifactList[ga][priority].keys(): gav = "%s:%s" % (ga, version) if maven_repo_util.somethingMatch(gavRegExps, gav): logging.debug("Dropping GAV %s:%s from priority %i because it matches an excluded " "GAV pattern.", ga, version, priority) del artifactList[ga][priority][version] else: artSpec = artifactList[ga][priority][version] for artType in copy.deepcopy(artSpec.artTypes.keys()): at = artSpec.artTypes[artType] for classifier in copy.deepcopy(at.classifiers): if classifier: gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, version) else: gatcv = "%s:%s:%s" % (ga, artType, version) if maven_repo_util.somethingMatch(gatcvRegExps, gatcv): logging.debug("Dropping GATCV %s from priority %i because it matches an excluded " "GAV pattern.", gatcv, priority) at.classifiers.remove(classifier) if not at.classifiers: logging.debug("Dropping GATV %s:%s:%s from priority %i because of no classifiers left.", ga, artType, version, priority) del artSpec.artTypes[artType] if not artSpec.containsMain(): logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.", ga, version, priority) del artifactList[ga][priority][version] if not artifactList[ga][priority]: logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority) del artifactList[ga][priority] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList
def _filterArtifactsByPatterns(self, artifacts, gavPatterns, gatcvs): if not gavPatterns and not gatcvs: return artifacts includedArtifacts = {} if gatcvs: for artifact in artifacts.keys(): artSpec = artifacts[artifact] artTypes = {} extContainsMain = False for ext in artSpec.artTypes.keys(): if ext == "pom": main = len(artSpec.artTypes.keys()) == 1 if not main: gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version) if gatcv in gatcvs: main = True extContainsMain = extContainsMain or main pomType = ArtifactType(ext, main, set([''])) artTypes[ext] = pomType else: main = False classifiers = set() for classifier in artSpec.artTypes[ext].classifiers: if classifier: gatcv = "%s:%s:%s:%s" % (artifact.getGA(), ext, classifier, artifact.version) else: gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version) if gatcv in gatcvs: classifiers.add(classifier) main = True else: if self._containedInAddClassifiers(ext, classifier): classifiers.add(classifier) extContainsMain = extContainsMain or main artType = ArtifactType(ext, main, classifiers) artTypes[ext] = artType if extContainsMain: artSpecToAdd = ArtifactSpec(artSpec.url, artTypes) includedArtifacts[artifact] = artSpecToAdd else: regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns) for artifact in artifacts.keys(): if maven_repo_util.somethingMatch(regExps, artifact.getGAV()): includedArtifacts[artifact] = artifacts[artifact] return includedArtifacts
def _filterExcludedGAVs(self, artifacts, excludedGAVs, priority): """ Filter artifactList removing specified GAVs. :param artifacts: artifact list to be filtered. :param excludedGAVs: list of excluded GAVs patterns :param priority: artifact source priority :returns: artifact list without artifacts that matched specified GAVs. """ logging.debug("Filtering excluded GAVs from partial result (priority %i).", priority) regExps = maven_repo_util.getRegExpsFromStrings(excludedGAVs) gavRegExps = [] gatcvRegExps = [] for regExp in regExps: if regExp.pattern.count(":") > 2: gatcvRegExps.append(regExp) else: gavRegExps.append(regExp) for artifact in copy.copy(artifacts): gav = artifact.getGAV() artSpec = artifacts[artifact] if maven_repo_util.somethingMatch(gavRegExps, gav): del artifacts[artifact] else: for artType in copy.deepcopy(artSpec.artTypes.keys()): at = artSpec.artTypes[artType] for classifier in copy.deepcopy(at.classifiers): ga = artifact.getGA() if classifier: gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, artifact.version) else: gatcv = "%s:%s:%s" % (ga, artType, artifact.version) if maven_repo_util.somethingMatch(gatcvRegExps, gatcv): logging.debug("Dropping GATCV %s because it matches an excluded GAV pattern.", gatcv) at.classifiers.remove(classifier) if not at.classifiers: logging.debug("Dropping GATV %s:%s:%s because of no classifiers left.", ga, artType, artifact.version) del artSpec.artTypes[artType] if not artSpec.containsMain(): logging.debug("Dropping GAV %s because of no main artifact left.", gav) del artifacts[artifact] return artifacts
def _filterExcludedGAVs(self, artifactList): """ Filter artifactList removing specified GAVs. :param artifactList: artifactList to be filtered. :returns: artifactList without arifacts that matched specified GAVs. """ logging.debug("Filtering artifacts with excluded GAVs.") regExps = maven_repo_util.getRegExpsFromStrings(self.config.excludedGAVs) for gat in artifactList.keys(): ga = gat.rpartition(':')[0] for priority in artifactList[gat].keys(): for version in artifactList[gat][priority].keys(): gav = ga + ":" + version if maven_repo_util.somethingMatch(regExps, gav): del artifactList[gat][priority][version] if not artifactList[gat][priority]: del artifactList[gat][priority] if not artifactList[gat]: del artifactList[gat] return artifactList
def _filterMultipleVersions(self, artifactList): logging.debug("Filtering multi-version artifacts to have just a single version.") regExps = maven_repo_util.getRegExpsFromStrings(self.config.multiVersionGAs, False) removeT = lambda gat: re.sub('\:[^:]+$', '', gat) for ga, gats in groupby(sorted(artifactList.keys()), removeT): gats = list(gats) if maven_repo_util.somethingMatch(regExps, ga): continue # Gather all priorities from all types priorities = set() for gat in gats: priorities.update(artifactList[gat].keys()) priorities = sorted(priorities) priority = priorities[0] # Gather all versions from all types versions = set() for gat in gats: # update versions only if this gat contains highest priority versions.update(artifactList[gat].get(priority, {}).keys()) versions = list(versions) if len(versions) > 1: # list of 1 is sorted by definition versions = maven_repo_util._sortVersionsWithAtlas(versions) # Remove version, priorities and gats from artifactList as necessary for gat in gats: for version in versions[1:]: artifactList[gat].get(priority, {}).pop(version, None) for p in priorities[1:]: artifactList[gat].pop(p, None) if not artifactList[gat] or not artifactList[gat][priority]: del artifactList[gat] # all versions or priorities were removed return artifactList
def _filterMultipleVersions(self, artifactList): logging.debug( "Filtering multi-version artifacts to have just a single version.") regExps = maven_repo_util.getRegExpsFromStrings( self.config.multiVersionGAs, False) for ga in sorted(artifactList.keys()): if maven_repo_util.somethingMatch(regExps, ga): continue # Gather all priorities priorities = sorted(artifactList[ga].keys()) priority = priorities[0] # Gather all versions versions = list(artifactList[ga][priority].keys()) if len(versions) > 1: # list of 1 is sorted by definition versions = maven_repo_util._sortVersionsWithAtlas(versions) # Remove version, priorities and gats from artifactList as necessary for version in versions[1:]: logging.debug( "Dropping GAV %s:%s from priority %i because only single version is allowed.", ga, version, priority) del artifactList[ga][priority][version] for p in priorities[1:]: logging.debug( "Dropping GA %s from priority %i because of no version left.", ga, p) del artifactList[ga][p] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList
def _filterExcludedGAVs(self, artifactList): """ Filter artifactList removing specified GAVs. :param artifactList: artifactList to be filtered. :returns: artifactList without artifacts that matched specified GAVs. """ logging.debug("Filtering artifacts with excluded GAVs.") regExps = maven_repo_util.getRegExpsFromStrings( self.config.excludedGAVs) gavRegExps = [] gatcvRegExps = [] for regExp in regExps: if regExp.pattern.count(":") > 2: gatcvRegExps.append(regExp) else: gavRegExps.append(regExp) for ga in artifactList.keys(): for priority in artifactList[ga].keys(): for version in artifactList[ga][priority].keys(): gav = "%s:%s" % (ga, version) if maven_repo_util.somethingMatch(gavRegExps, gav): logging.debug( "Dropping GAV %s:%s from priority %i because it matches an excluded " "GAV pattern.", ga, version, priority) del artifactList[ga][priority][version] else: artSpec = artifactList[ga][priority][version] for artType in copy.deepcopy(artSpec.artTypes.keys()): at = artSpec.artTypes[artType] for classifier in copy.deepcopy(at.classifiers): if classifier: gatcv = "%s:%s:%s:%s" % ( ga, artType, classifier, version) else: gatcv = "%s:%s:%s" % (ga, artType, version) if maven_repo_util.somethingMatch( gatcvRegExps, gatcv): logging.debug( "Dropping GATCV %s from priority %i because it matches an excluded " "GAV pattern.", gatcv, priority) at.classifiers.remove(classifier) if not at.classifiers: logging.debug( "Dropping GATV %s:%s:%s from priority %i because of no classifiers left.", ga, artType, version, priority) del artSpec.artTypes[artType] if not artSpec.containsMain(): logging.debug( "Dropping GAV %s:%s from priority %i because of no main artifact left.", ga, version, priority) del artifactList[ga][priority][version] if not artifactList[ga][priority]: logging.debug( "Dropping GA %s from priority %i because of no version left.", ga, priority) del artifactList[ga][priority] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList
def _filterExcludedTypes(self, artifactList): ''' Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with not-excluded artifact types only. :param artifactList: artifactList to be filtered. :param exclTypes: list of excluded types :returns: artifactList without artifacts that matched specified types and had no other main types. ''' logging.debug("Filtering artifacts with excluded types.") regExps = maven_repo_util.getRegExpsFromStrings( self.config.gatcvWhitelist) exclTypes = self.config.excludedTypes for ga in artifactList.keys(): for priority in artifactList[ga].keys(): for version in artifactList[ga][priority].keys(): artSpec = artifactList[ga][priority][version] for artType in list(artSpec.artTypes.keys()): if artType in exclTypes: artTypeObj = artSpec.artTypes[artType] classifiers = artTypeObj.classifiers (groupId, artifactId) = ga.split(':') for classifier in list(classifiers): art = MavenArtifact(groupId, artifactId, artType, version, classifier) gatcv = art.getGATCV() if not maven_repo_util.somethingMatch( regExps, gatcv): logging.debug( "Dropping classifier \"%s\" of %s:%s:%s from priority %i because of " "excluded type.", classifier, ga, artType, version, priority) classifiers.remove(classifier) else: logging.debug( "Skipping drop of %s:%s:%s:%s from priority %i because it matches a " "whitelist pattern.", ga, artType, classifier, version, priority) if not classifiers: logging.debug( "Dropping %s:%s:%s from priority %i because of no classifier left.", ga, artType, version, priority) del (artSpec.artTypes[artType]) noMain = True for artType in artSpec.artTypes.keys(): artTypeObj = artSpec.artTypes[artType] if artTypeObj.mainType: noMain = False break if not artSpec.artTypes or noMain: if noMain: logging.debug( "Dropping GAV %s:%s from priority %i because of no main artifact left.", ga, version, priority) else: logging.debug( "Dropping GAV %s:%s from priority %i because of no artifact type left.", ga, version, priority) del artifactList[ga][priority][version] if not artifactList[ga][priority]: logging.debug( "Dropping GA %s from priority %i because of no version left.", ga, priority) del artifactList[ga][priority] if not artifactList[ga]: logging.debug("Dropping GA %s because of no priority left.", ga) del artifactList[ga] return artifactList