def _filterMultipleVersions(self, artifactList):
logging.debug("Filtering multi-version artifacts to have just a single version.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.multiVersionGAs, False)
for ga in sorted(artifactList.keys()):
if maven_repo_util.somethingMatch(regExps, ga):
continue
# Gather all priorities
priorities = sorted(artifactList[ga].keys())
priority = priorities[0]
# Gather all versions
versions = list(artifactList[ga][priority].keys())
if len(versions) > 1: # list of 1 is sorted by definition
versions = maven_repo_util._sortVersionsWithAtlas(versions)
# Remove version, priorities and gats from artifactList as necessary
for version in versions[1:]:
logging.debug("Dropping GAV %s:%s from priority %i because only single version is allowed.", ga,
version, priority)
del artifactList[ga][priority][version]
for p in priorities[1:]:
logging.debug("Dropping GA %s from priority %i because of no version left.", ga, p)
del artifactList[ga][p]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.", ga)
del artifactList[ga]
return artifactList
def _filterExcludedTypes(self, artifactList):
'''
Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
not-excluded artifact types only.
:param artifactList: artifactList to be filtered.
:param exclTypes: list of excluded types
:returns: artifactList without artifacts that matched specified types and had no other main types.
'''
logging.debug("Filtering artifacts with excluded types.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.gatcvWhitelist)
exclTypes = self.config.excludedTypes
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artSpec = artifactList[ga][priority][version]
for artType in list(artSpec.artTypes.keys()):
if artType in exclTypes:
artTypeObj = artSpec.artTypes[artType]
classifiers = artTypeObj.classifiers
(groupId, artifactId) = ga.split(':')
for classifier in list(classifiers):
art = MavenArtifact(groupId, artifactId, artType, version, classifier)
gatcv = art.getGATCV()
if not maven_repo_util.somethingMatch(regExps, gatcv):
logging.debug("Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
"excluded type.", classifier, ga, artType, version, priority)
classifiers.remove(classifier)
else:
logging.debug("Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
"whitelist pattern.", ga, artType, classifier, version, priority)
if not classifiers:
logging.debug("Dropping %s:%s:%s from priority %i because of no classifier left.", ga,
artType, version, priority)
del(artSpec.artTypes[artType])
noMain = True
for artType in artSpec.artTypes.keys():
artTypeObj = artSpec.artTypes[artType]
if artTypeObj.mainType:
noMain = False
break
if not artSpec.artTypes or noMain:
if noMain:
logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
else:
logging.debug("Dropping GAV %s:%s from priority %i because of no artifact type left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.", ga)
del artifactList[ga]
return artifactList
def _filterArtifactsByPatterns(self, artifacts, gavPatterns):
if not gavPatterns:
return artifacts
regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns)
includedArtifacts = {}
for artifact in artifacts:
if maven_repo_util.somethingMatch(regExps, artifact.getGAV()):
includedArtifacts[artifact] = artifacts[artifact]
return includedArtifacts
def _filterArtifactsByPatterns(self, artifacts, gavPatterns):
if not gavPatterns:
return artifacts
regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns)
includedArtifacts = {}
for artifact in artifacts:
if maven_repo_util.somethingMatch(regExps, artifact.getGAV()):
includedArtifacts[artifact] = artifacts[artifact]
return includedArtifacts
def _filterExcludedGAVs(self, artifactList):
"""
Filter artifactList removing specified GAVs.
:param artifactList: artifactList to be filtered.
:returns: artifactList without artifacts that matched specified GAVs.
"""
logging.debug("Filtering artifacts with excluded GAVs.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.excludedGAVs)
gavRegExps = []
gatcvRegExps = []
for regExp in regExps:
if regExp.pattern.count(":") > 2:
gatcvRegExps.append(regExp)
else:
gavRegExps.append(regExp)
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
gav = "%s:%s" % (ga, version)
if maven_repo_util.somethingMatch(gavRegExps, gav):
logging.debug("Dropping GAV %s:%s from priority %i because it matches an excluded "
"GAV pattern.", ga, version, priority)
del artifactList[ga][priority][version]
else:
artSpec = artifactList[ga][priority][version]
for artType in copy.deepcopy(artSpec.artTypes.keys()):
at = artSpec.artTypes[artType]
for classifier in copy.deepcopy(at.classifiers):
if classifier:
gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, version)
else:
gatcv = "%s:%s:%s" % (ga, artType, version)
if maven_repo_util.somethingMatch(gatcvRegExps, gatcv):
logging.debug("Dropping GATCV %s from priority %i because it matches an excluded "
"GAV pattern.", gatcv, priority)
at.classifiers.remove(classifier)
if not at.classifiers:
logging.debug("Dropping GATV %s:%s:%s from priority %i because of no classifiers left.",
ga, artType, version, priority)
del artSpec.artTypes[artType]
if not artSpec.containsMain():
logging.debug("Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug("Dropping GA %s from priority %i because of no version left.", ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.", ga)
del artifactList[ga]
return artifactList
def _filterArtifactsByPatterns(self, artifacts, gavPatterns, gatcvs):
if not gavPatterns and not gatcvs:
return artifacts
includedArtifacts = {}
if gatcvs:
for artifact in artifacts.keys():
artSpec = artifacts[artifact]
artTypes = {}
extContainsMain = False
for ext in artSpec.artTypes.keys():
if ext == "pom":
main = len(artSpec.artTypes.keys()) == 1
if not main:
gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version)
if gatcv in gatcvs:
main = True
extContainsMain = extContainsMain or main
pomType = ArtifactType(ext, main, set(['']))
artTypes[ext] = pomType
else:
main = False
classifiers = set()
for classifier in artSpec.artTypes[ext].classifiers:
if classifier:
gatcv = "%s:%s:%s:%s" % (artifact.getGA(), ext, classifier, artifact.version)
else:
gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version)
if gatcv in gatcvs:
classifiers.add(classifier)
main = True
else:
if self._containedInAddClassifiers(ext, classifier):
classifiers.add(classifier)
extContainsMain = extContainsMain or main
artType = ArtifactType(ext, main, classifiers)
artTypes[ext] = artType
if extContainsMain:
artSpecToAdd = ArtifactSpec(artSpec.url, artTypes)
includedArtifacts[artifact] = artSpecToAdd
else:
regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns)
for artifact in artifacts.keys():
if maven_repo_util.somethingMatch(regExps, artifact.getGAV()):
includedArtifacts[artifact] = artifacts[artifact]
return includedArtifacts
def _filterArtifactsByPatterns(self, artifacts, gavPatterns, gatcvs):
if not gavPatterns and not gatcvs:
return artifacts
includedArtifacts = {}
if gatcvs:
for artifact in artifacts.keys():
artSpec = artifacts[artifact]
artTypes = {}
extContainsMain = False
for ext in artSpec.artTypes.keys():
if ext == "pom":
main = len(artSpec.artTypes.keys()) == 1
if not main:
gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version)
if gatcv in gatcvs:
main = True
extContainsMain = extContainsMain or main
pomType = ArtifactType(ext, main, set(['']))
artTypes[ext] = pomType
else:
main = False
classifiers = set()
for classifier in artSpec.artTypes[ext].classifiers:
if classifier:
gatcv = "%s:%s:%s:%s" % (artifact.getGA(), ext, classifier, artifact.version)
else:
gatcv = "%s:%s:%s" % (artifact.getGA(), ext, artifact.version)
if gatcv in gatcvs:
classifiers.add(classifier)
main = True
else:
if self._containedInAddClassifiers(ext, classifier):
classifiers.add(classifier)
extContainsMain = extContainsMain or main
artType = ArtifactType(ext, main, classifiers)
artTypes[ext] = artType
if extContainsMain:
artSpecToAdd = ArtifactSpec(artSpec.url, artTypes)
includedArtifacts[artifact] = artSpecToAdd
else:
regExps = maven_repo_util.getRegExpsFromStrings(gavPatterns)
for artifact in artifacts.keys():
if maven_repo_util.somethingMatch(regExps, artifact.getGAV()):
includedArtifacts[artifact] = artifacts[artifact]
return includedArtifacts
def _filterExcludedGAVs(self, artifacts, excludedGAVs, priority):
"""
Filter artifactList removing specified GAVs.
:param artifacts: artifact list to be filtered.
:param excludedGAVs: list of excluded GAVs patterns
:param priority: artifact source priority
:returns: artifact list without artifacts that matched specified GAVs.
"""
logging.debug("Filtering excluded GAVs from partial result (priority %i).", priority)
regExps = maven_repo_util.getRegExpsFromStrings(excludedGAVs)
gavRegExps = []
gatcvRegExps = []
for regExp in regExps:
if regExp.pattern.count(":") > 2:
gatcvRegExps.append(regExp)
else:
gavRegExps.append(regExp)
for artifact in copy.copy(artifacts):
gav = artifact.getGAV()
artSpec = artifacts[artifact]
if maven_repo_util.somethingMatch(gavRegExps, gav):
del artifacts[artifact]
else:
for artType in copy.deepcopy(artSpec.artTypes.keys()):
at = artSpec.artTypes[artType]
for classifier in copy.deepcopy(at.classifiers):
ga = artifact.getGA()
if classifier:
gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, artifact.version)
else:
gatcv = "%s:%s:%s" % (ga, artType, artifact.version)
if maven_repo_util.somethingMatch(gatcvRegExps, gatcv):
logging.debug("Dropping GATCV %s because it matches an excluded GAV pattern.", gatcv)
at.classifiers.remove(classifier)
if not at.classifiers:
logging.debug("Dropping GATV %s:%s:%s because of no classifiers left.", ga, artType,
artifact.version)
del artSpec.artTypes[artType]
if not artSpec.containsMain():
logging.debug("Dropping GAV %s because of no main artifact left.", gav)
del artifacts[artifact]
return artifacts
def _filterExcludedGAVs(self, artifacts, excludedGAVs, priority):
"""
Filter artifactList removing specified GAVs.
:param artifacts: artifact list to be filtered.
:param excludedGAVs: list of excluded GAVs patterns
:param priority: artifact source priority
:returns: artifact list without artifacts that matched specified GAVs.
"""
logging.debug("Filtering excluded GAVs from partial result (priority %i).", priority)
regExps = maven_repo_util.getRegExpsFromStrings(excludedGAVs)
gavRegExps = []
gatcvRegExps = []
for regExp in regExps:
if regExp.pattern.count(":") > 2:
gatcvRegExps.append(regExp)
else:
gavRegExps.append(regExp)
for artifact in copy.copy(artifacts):
gav = artifact.getGAV()
artSpec = artifacts[artifact]
if maven_repo_util.somethingMatch(gavRegExps, gav):
del artifacts[artifact]
else:
for artType in copy.deepcopy(artSpec.artTypes.keys()):
at = artSpec.artTypes[artType]
for classifier in copy.deepcopy(at.classifiers):
ga = artifact.getGA()
if classifier:
gatcv = "%s:%s:%s:%s" % (ga, artType, classifier, artifact.version)
else:
gatcv = "%s:%s:%s" % (ga, artType, artifact.version)
if maven_repo_util.somethingMatch(gatcvRegExps, gatcv):
logging.debug("Dropping GATCV %s because it matches an excluded GAV pattern.", gatcv)
at.classifiers.remove(classifier)
if not at.classifiers:
logging.debug("Dropping GATV %s:%s:%s because of no classifiers left.", ga, artType,
artifact.version)
del artSpec.artTypes[artType]
if not artSpec.containsMain():
logging.debug("Dropping GAV %s because of no main artifact left.", gav)
del artifacts[artifact]
return artifacts
def _filterExcludedGAVs(self, artifactList):
"""
Filter artifactList removing specified GAVs.
:param artifactList: artifactList to be filtered.
:returns: artifactList without arifacts that matched specified GAVs.
"""
logging.debug("Filtering artifacts with excluded GAVs.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.excludedGAVs)
for gat in artifactList.keys():
ga = gat.rpartition(':')[0]
for priority in artifactList[gat].keys():
for version in artifactList[gat][priority].keys():
gav = ga + ":" + version
if maven_repo_util.somethingMatch(regExps, gav):
del artifactList[gat][priority][version]
if not artifactList[gat][priority]:
del artifactList[gat][priority]
if not artifactList[gat]:
del artifactList[gat]
return artifactList
def _filterMultipleVersions(self, artifactList):
logging.debug("Filtering multi-version artifacts to have just a single version.")
regExps = maven_repo_util.getRegExpsFromStrings(self.config.multiVersionGAs, False)
removeT = lambda gat: re.sub('\:[^:]+$', '', gat)
for ga, gats in groupby(sorted(artifactList.keys()), removeT):
gats = list(gats)
if maven_repo_util.somethingMatch(regExps, ga):
continue
# Gather all priorities from all types
priorities = set()
for gat in gats:
priorities.update(artifactList[gat].keys())
priorities = sorted(priorities)
priority = priorities[0]
# Gather all versions from all types
versions = set()
for gat in gats:
# update versions only if this gat contains highest priority
versions.update(artifactList[gat].get(priority, {}).keys())
versions = list(versions)
if len(versions) > 1: # list of 1 is sorted by definition
versions = maven_repo_util._sortVersionsWithAtlas(versions)
# Remove version, priorities and gats from artifactList as necessary
for gat in gats:
for version in versions[1:]:
artifactList[gat].get(priority, {}).pop(version, None)
for p in priorities[1:]:
artifactList[gat].pop(p, None)
if not artifactList[gat] or not artifactList[gat][priority]:
del artifactList[gat] # all versions or priorities were removed
return artifactList
def _filterMultipleVersions(self, artifactList):
logging.debug(
"Filtering multi-version artifacts to have just a single version.")
regExps = maven_repo_util.getRegExpsFromStrings(
self.config.multiVersionGAs, False)
for ga in sorted(artifactList.keys()):
if maven_repo_util.somethingMatch(regExps, ga):
continue
# Gather all priorities
priorities = sorted(artifactList[ga].keys())
priority = priorities[0]
# Gather all versions
versions = list(artifactList[ga][priority].keys())
if len(versions) > 1: # list of 1 is sorted by definition
versions = maven_repo_util._sortVersionsWithAtlas(versions)
# Remove version, priorities and gats from artifactList as necessary
for version in versions[1:]:
logging.debug(
"Dropping GAV %s:%s from priority %i because only single version is allowed.",
ga, version, priority)
del artifactList[ga][priority][version]
for p in priorities[1:]:
logging.debug(
"Dropping GA %s from priority %i because of no version left.",
ga, p)
del artifactList[ga][p]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.",
ga)
del artifactList[ga]
return artifactList
def _filterExcludedGAVs(self, artifactList):
"""
Filter artifactList removing specified GAVs.
:param artifactList: artifactList to be filtered.
:returns: artifactList without artifacts that matched specified GAVs.
"""
logging.debug("Filtering artifacts with excluded GAVs.")
regExps = maven_repo_util.getRegExpsFromStrings(
self.config.excludedGAVs)
gavRegExps = []
gatcvRegExps = []
for regExp in regExps:
if regExp.pattern.count(":") > 2:
gatcvRegExps.append(regExp)
else:
gavRegExps.append(regExp)
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
gav = "%s:%s" % (ga, version)
if maven_repo_util.somethingMatch(gavRegExps, gav):
logging.debug(
"Dropping GAV %s:%s from priority %i because it matches an excluded "
"GAV pattern.", ga, version, priority)
del artifactList[ga][priority][version]
else:
artSpec = artifactList[ga][priority][version]
for artType in copy.deepcopy(artSpec.artTypes.keys()):
at = artSpec.artTypes[artType]
for classifier in copy.deepcopy(at.classifiers):
if classifier:
gatcv = "%s:%s:%s:%s" % (
ga, artType, classifier, version)
else:
gatcv = "%s:%s:%s" % (ga, artType, version)
if maven_repo_util.somethingMatch(
gatcvRegExps, gatcv):
logging.debug(
"Dropping GATCV %s from priority %i because it matches an excluded "
"GAV pattern.", gatcv, priority)
at.classifiers.remove(classifier)
if not at.classifiers:
logging.debug(
"Dropping GATV %s:%s:%s from priority %i because of no classifiers left.",
ga, artType, version, priority)
del artSpec.artTypes[artType]
if not artSpec.containsMain():
logging.debug(
"Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug(
"Dropping GA %s from priority %i because of no version left.",
ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.",
ga)
del artifactList[ga]
return artifactList
def _filterExcludedTypes(self, artifactList):
'''
Filter artifactList removing GAVs with specified main types only, otherwise keeping GAVs with
not-excluded artifact types only.
:param artifactList: artifactList to be filtered.
:param exclTypes: list of excluded types
:returns: artifactList without artifacts that matched specified types and had no other main types.
'''
logging.debug("Filtering artifacts with excluded types.")
regExps = maven_repo_util.getRegExpsFromStrings(
self.config.gatcvWhitelist)
exclTypes = self.config.excludedTypes
for ga in artifactList.keys():
for priority in artifactList[ga].keys():
for version in artifactList[ga][priority].keys():
artSpec = artifactList[ga][priority][version]
for artType in list(artSpec.artTypes.keys()):
if artType in exclTypes:
artTypeObj = artSpec.artTypes[artType]
classifiers = artTypeObj.classifiers
(groupId, artifactId) = ga.split(':')
for classifier in list(classifiers):
art = MavenArtifact(groupId, artifactId,
artType, version,
classifier)
gatcv = art.getGATCV()
if not maven_repo_util.somethingMatch(
regExps, gatcv):
logging.debug(
"Dropping classifier \"%s\" of %s:%s:%s from priority %i because of "
"excluded type.", classifier, ga,
artType, version, priority)
classifiers.remove(classifier)
else:
logging.debug(
"Skipping drop of %s:%s:%s:%s from priority %i because it matches a "
"whitelist pattern.", ga, artType,
classifier, version, priority)
if not classifiers:
logging.debug(
"Dropping %s:%s:%s from priority %i because of no classifier left.",
ga, artType, version, priority)
del (artSpec.artTypes[artType])
noMain = True
for artType in artSpec.artTypes.keys():
artTypeObj = artSpec.artTypes[artType]
if artTypeObj.mainType:
noMain = False
break
if not artSpec.artTypes or noMain:
if noMain:
logging.debug(
"Dropping GAV %s:%s from priority %i because of no main artifact left.",
ga, version, priority)
else:
logging.debug(
"Dropping GAV %s:%s from priority %i because of no artifact type left.",
ga, version, priority)
del artifactList[ga][priority][version]
if not artifactList[ga][priority]:
logging.debug(
"Dropping GA %s from priority %i because of no version left.",
ga, priority)
del artifactList[ga][priority]
if not artifactList[ga]:
logging.debug("Dropping GA %s because of no priority left.",
ga)
del artifactList[ga]
return artifactList
