def __printSection(self, new, untouched, revoked, all, title, path,
summary):
output = []
removeline = "Revoked/Replaced/Renamed"
totaline = "Total (New + Untouched + Revoked/Replaced/Renamed)"
caps = title.upper()
if title == "supplemental":
removeline = "Removed"
totaline = "Total"
title = title + " files"
caps = title.upper()
elif title == 'gbdbs':
caps = "GBDBS"
title = "gbdb files"
elif title == "download":
title = title + " files"
caps = title.upper()
if all:
output.append("")
output.append("%s:" % caps)
output.append("New: %s" % len(new))
output.append("Untouched: %s" % len(untouched))
output.append("%s: %s" % (removeline, len(revoked)))
output.append("New + Untouched: %s" % len(new | untouched))
output.append("%s: %s" % (totaline, len(all)))
intersect = new & revoked
if intersect:
output.append("")
output.append(
"These %s objects exist in both new and revoked %s:" %
(len(intersect), title))
sys.stderr.write(
"\n\nWARNING: There are objects that are both new and revoked. Those files should be removed from the new release directory, as they haven't been released yet.\n\n"
)
for i in intersect:
output.append("%s" % i)
if all and not summary:
output.append("")
output.append("New %s (%s):" % (title.title(), len(new)))
output.extend(ucscUtils.printIter(new, path))
output.append("")
output.append("Untouched %s (%s):" %
(title.title(), len(untouched)))
output.extend(ucscUtils.printIter(untouched, path))
output.append("")
output.append("%s %s (%s):" %
(removeline, title.title(), len(revoked)))
output.extend(ucscUtils.printIter(revoked, path))
if all:
output.append("")
return output
def printReportOne(self, args, c):
(totalFiles, revokedFiles, newGbdbSet, revokedGbdbs, newTableSet, revokedTables, additionalList, atticSet, newSupplementalSet, tableSize) = (self.totalFiles, self.revokedFiles, self.newGbdbSet, self.revokedGbdbs, self.newTableSet, self.revokedTableSet, self.additionalList, self.atticSet, self.newSupplementalSet, self.tableSize)
output = []
newTables = newTableSet - revokedTables
newFiles = totalFiles - revokedFiles
newGbdbs = newGbdbSet - revokedGbdbs
output.extend(self.__qaHeader(output, newTables, newFiles, newGbdbSet, newSupplementalSet, set(), additionalList, revokedTables, revokedFiles, revokedGbdbs, totalFiles, newGbdbSet, args, c))
self.newTables = set(newTables)
self.newFiles = set(ucscUtils.printIter(newFiles, self.releasePath))
self.newGbdbs = set(ucscUtils.printIter(newGbdbs, self.gbdbPath))
self.newSupplemental = set(ucscUtils.printIter(newSupplementalSet, self.releasePath))
self.newOthers = set(ucscUtils.printIter(additionalList, self.releasePath))
if not args['summary']:
output.append("")
output.extend(self.__printSectionOne(output, self.newTables, "New Tables"))
output.extend(self.__printSectionOne(output, self.newFiles, "New Download Files"))
output.extend(self.__printSectionOne(output, self.newGbdbs, "New Gbdb Files"))
output.extend(self.__printSectionOne(output, self.newSupplemental, "New Supplemental Files"))
output.extend(self.__printSectionOne(output, self.newOthers, "New Other Files"))
output.extend(self.__printSectionOne(output, ucscUtils.printIter(revokedTables, 0), "Revoked Tables"))
output.extend(self.__printSectionOne(output, ucscUtils.printIter(revokedFiles, self.releasePath), "Revoked Files"))
output.extend(self.__printSectionOne(output, ucscUtils.printIter(revokedGbdbs, self.gbdbPath), "Revoked Gbdbs"))
if self.atticSet:
output.append("Attic Objects")
output.extend(ucscUtils.printIter((self.atticSet), self.releasePath))
if not args['ignore']:
output.append("No Errors")
else:
output.append("The counts here were generated by ignoring errors, they may not be correct")
return output
def __printSection(self, new, untouched, revoked, all, title, path,
summary):
output = []
removeline = "Revoked/Replaced/Renamed"
totaline = "Total (New + Untouched + Revoked/Replaced/Renamed)"
caps = title.upper()
if title == "supplemental":
removeline = "Removed"
totaline = "Total"
title = title + " files"
caps = title.upper()
elif title == 'gbdbs':
caps = "GBDBS"
title = "gbdb files"
elif title == "download":
title = title + " files"
caps = title.upper()
if all:
output.append("")
output.append("%s:" % caps)
output.append("New: %s" % len(new))
output.append("Untouched: %s" % len(untouched))
output.append("%s: %s" % (removeline, len(revoked)))
output.append("New + Untouched: %s" % len(new | untouched))
output.append("%s: %s" % (totaline, len(all)))
intersect = new & revoked
if intersect:
output.append("")
output.append(
"These %s objects exist in both new and revoked %s:" %
(len(intersect), title))
for i in intersect:
output.append("%s" % i)
if all and not summary:
output.append("")
output.append("New %s (%s):" % (title.title(), len(new)))
output.extend(ucscUtils.printIter(new, path))
output.append("")
output.append("Untouched %s (%s):" %
(title.title(), len(untouched)))
output.extend(ucscUtils.printIter(untouched, path))
output.append("")
output.append("%s %s (%s):" %
(removeline, title.title(), len(revoked)))
output.extend(ucscUtils.printIter(revoked, path))
if all:
output.append("")
return output
def __addMissingToReport(self, missing, type, path=None):
output = []
if missing:
output.append("%s that dropped between releases (%s):" % (type, len(missing)))
output.extend(ucscUtils.printIter(missing, path))
output.append("\n")
return output
def __addMissingToReport(self, missing, type, path=None):
output = []
if missing:
output.append("%s that dropped between releases (%s):" %
(type, len(missing)))
output.extend(ucscUtils.printIter(missing, path))
output.append("\n")
return output
def __printSection(self, new, untouched, revoked, all, title, path, summary):
output = []
removeline = "Revoked/Replaced/Renamed"
totaline = "Total (New + Untouched + Revoked/Replaced/Renamed)"
caps = title.upper()
if title == "supplemental":
removeline = "Removed"
totaline = "Total"
title = title + " files"
caps = title.upper()
elif title == 'gbdbs':
caps = "GBDBS"
title = "gbdb files"
elif title == "download":
title = title + " files"
caps = title.upper()
if all:
output.append("")
output.append("%s:" % caps)
output.append("New: %s" % len(new))
output.append("Untouched: %s" % len(untouched))
output.append("%s: %s" % (removeline, len(revoked)))
output.append("New + Untouched: %s" % len(new | untouched))
output.append("%s: %s" % (totaline, len(all)))
intersect = new & revoked
if intersect:
output.append("")
output.append("These %s objects exist in both new and revoked %s:" % (len(intersect), title))
sys.stderr.write("\n\nWARNING: There are objects that are both new and revoked. Those files should be removed from the new release directory, as they haven't been released yet.\n\n")
for i in intersect:
output.append("%s" % i)
if all and not summary:
output.append("")
output.append("New %s (%s):" % (title.title(), len(new)))
output.extend(ucscUtils.printIter(new, path))
output.append("")
output.append("Untouched %s (%s):" % (title.title(), len(untouched)))
output.extend(ucscUtils.printIter(untouched, path))
output.append("")
output.append("%s %s (%s):" % (removeline, title.title(), len(revoked)))
output.extend(ucscUtils.printIter(revoked, path))
if all:
output.append("")
return output
def __init__(self, args):
self.releaseNew = args['releaseNew']
self.releaseOld = args['releaseOld']
self.database = args['database']
self.composite = args['composite']
self.loose = args['loose']
self.ignore = args['ignore']
self.summary = args['summary']
self.specialMdb = args['specialMdb']
self.args = args
if 'verbose' in args:
self.verbose = args['verbose']
else:
self.verbose = 0
errors = []
c = track.CompositeTrack(self.database, self.composite, None, self.specialMdb)
#sanitize arguments
if not self.releaseOld.isdigit():
self.releaseOld = 'solo'
elif int(self.releaseOld) <= 0:
self.releaseOlf = 'solo'
elif self.releaseOld > self.releaseNew:
self.releaseOld = 'solo'
if self.verbose >= 1:
sys.stderr.write("Initializing MkChangeNotes\n")
self.releasePath = c.httpDownloadsPath + 'release' + args['releaseNew']
self.gbdbPath = "/gbdb/%s/bbi" % args['database']
self.trackDbFile = c.currentTrackDb
if not self.trackDbFile:
self.trackDb = None
errors.append("track: There is no entry in trackDb.wgEncode.ra for %s with the alpha tag" % self.composite)
else:
self.trackDb = RaFile(self.trackDbFile, "track")
if int(self.releaseNew) > 1 and str(self.releaseOld) != 'solo':
if self.verbose >= 2:
sys.stderr.write("Comparison mode\n")
self.newReleaseFiles = c.releases[int(self.releaseNew)-1]
self.oldReleaseFiles = c.releases[int(self.releaseOld)-1]
self.releasePathOld = c.httpDownloadsPath + 'release' + args['releaseOld']
self.newMdb = c.alphaMetaDb
self.oldMdb = c.publicMetaDb
if self.verbose >= 2:
sys.stderr.write("Checking for missing files\n")
#make a list of missing files
self.missingFiles = self.__checkFilesForDropped()
#filter them out of old release files
if self.verbose >= 1:
sys.stderr.write("Scanning and parsing release directories\n")
#check if all files listed in release directories have associated metaDb entries
(self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet, self.newSupplementalSet, newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new")
(self.oldMdb, self.oldRevokedSet, self.oldRevokedFiles, self.oldAtticSet, self.oldSupplementalSet, oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old")
self.expIds = set(self.newMdb.filter(lambda s: 'expId' in s, lambda s: s['expId']))
if self.verbose >= 2:
sys.stderr.write("Checking for attic files\n")
#check that attic fiels aren't in trackDb
if self.trackDb:
errors.extend(self.__checkAtticNotInTrackDb())
#checks to see that nothing has disappeared between public and alpha
if self.verbose >= 1:
sys.stderr.write("Checking new metaDb for missing stanzas\n")
errors.extend(self.__checkAlphaForDropped("alpha metaDb", "stanza"))
if self.verbose >=1:
sys.stderr.write("Checking file md5sums across releases\n")
errors.extend(self.__checkMd5sums())
#checks and gets tables that are present, also returns a revoked set of tables for new
if self.verbose >= 1:
sys.stderr.write("Checking table status\n")
(self.newTableSet, self.revokedTableSet, self.newMissingTables, newTableError) = self.checkTableStatus("alpha metaDb", "new")
(self.oldTableSet, spam, self.droppedTables, oldTableError) = self.checkTableStatus("public metaDb", "old")
self.newInAttic = self.atticSet - self.oldAtticSet
self.stillInAttic = self.oldAtticSet & self.atticSet
self.oldTableSet = self.oldTableSet - self.atticSet
self.noMoreAttic = self.oldAtticSet - self.atticSet
self.changedTables = self.oldTableSet - self.newTableSet - self.revokedTableSet
#same as above except for gbdbs
if self.verbose >= 1:
sys.stderr.write("Checking GBDB status\n")
(self.newGbdbSet, self.revokedGbdbs, newGbdbError) = self.getGbdbFiles("new")
(self.oldGbdbSet, eggs, oldGbdbError) = self.getGbdbFiles("old")
#remove missing files from gbdbs
self.oldGbdbSet = self.oldGbdbSet - self.missingFiles
self.oldGbdbSet = self.oldGbdbSet - self.atticSet
self.changedGbdbs = self.oldGbdbSet - self.newGbdbSet - self.revokedGbdbs
for i in self.missingFiles:
if i in self.oldReleaseFiles:
del self.oldReleaseFiles[i]
#fill in the errors
errors.extend(newFileErrors)
errors.extend(oldFileErrors)
errors.extend(newTableError)
errors.extend(oldTableError)
errors.extend(newGbdbError)
errors.extend(oldGbdbError)
if self.changedTables:
errors.append("These tables were tables in the old release, but are no longer tables in the new release:")
errors.extend(list(self.changedTables))
if self.changedGbdbs:
errors.append("These GBDBs were GBDB tables in the old release, but are no longer GBDB tables in the new release:")
errors.extend(list(self.changedGbdbs))
#for ease of typing
totalFiles = set(self.newReleaseFiles)
oldTotalFiles = set(self.oldReleaseFiles)
#these could honestly be moved earlier, get a file list processing section or something
#they clean out special fiels out and separated the master fiels list into the 3 required
#ones: wgEncode, supplemental and additional.
self.totalFiles = self.__cleanSpecialFiles(totalFiles)
self.oldTotalFiles = self.__cleanSpecialFiles(oldTotalFiles)
(self.oldTotalFiles, self.additionalList, self.oldAdditionalList, self.totalFiles) = self.__separateOutAdditional()
#get the stuff you need to push
self.pushTables = set(sorted((self.newTableSet - self.oldTableSet)))
self.pushFiles = set(sorted((self.totalFiles - self.oldTotalFiles)))
self.pushGbdbs = set(sorted((self.newGbdbSet - self.oldGbdbSet)))
self.newSupp = self.newSupplementalSet - self.oldSupplementalSet
self.newTables = set(self.pushTables)
self.newFiles = set(ucscUtils.printIter(self.pushFiles, self.releasePath))
self.newGbdbs = set(ucscUtils.printIter(self.pushGbdbs, self.gbdbPath))
self.newSupplemental = set(ucscUtils.printIter(self.newSupp, self.releasePath))
self.newOthers = set(ucscUtils.printIter(self.additionalList, self.releasePath))
self.fullFiles = sorted(self.totalFiles - self.revokedFiles)
self.fullTables = self.oldTableSet & self.newTableSet
self.errors = errors
#don't output.append(report unless ignore option is on or no errors
#module mode doesn't generate output by default
if self.verbose >= 1:
sys.stderr.write("Creating report\n")
if (not errors) or self.ignore:
self.output = self.printReport(args, c)
else:
self.output = self.printErrors(errors, self.missingFiles)
elif self.releaseOld == 'solo':
self.newReleaseFiles = c.releases[int(self.releaseNew)-1]
self.oldReleaseFiles = set()
self.newMdb = c.alphaMetaDb
#check that attic fiels aren't in trackDb
if self.trackDb:
errors.extend(self.__checkAtticNotInTrackDb())
(self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet, self.newSupplementalSet, newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new")
self.expIds = set(self.newMdb.filter(lambda s: 'expId' in s, lambda s: s['expId']))
(self.newTableSet, self.revokedTableSet, spam, newTableError) = self.checkTableStatus("alpha metaDb", "new")
self.tableSize = self.__getTableSize()
(self.newGbdbSet, self.revokedGbdbs, newGbdbError) = self.getGbdbFiles("new")
#collect errors
errors.extend(newFileErrors)
errors.extend(newTableError)
errors.extend(newGbdbError)
#set for easy operations
totalFiles = set(self.newReleaseFiles)
#clean out special fiels we don't push i.e. md5sum.history
self.totalFiles = self.__cleanSpecialFiles(totalFiles)
self.pushTables = self.newTableSet
self.pushFiles = self.totalFiles
self.pushGbdbs = self.newGbdbSet
self.newSupp = self.newSupplementalSet
self.fullFiles = self.totalFiles
self.fullTables = self.newTableSet
#makes list for additional files
(self.oldTotalFiles, self.oldSupplementalSet) = (set(), set())
(self.oldReleaseFiles, self.additionalList, self.oldAdditionalList, self.totalFiles) = self.__separateOutAdditional()
self.errors = errors
if (not errors) or self.ignore:
self.output = self.printReportOne(args, c)
else:
self.droppedTables = set()
self.output = self.printErrors(errors, set())
def printReport(self, args, c):
(totalFiles, newGbdbSet, newTableSet, additionalList, oldAdditionalList, oldTableSet, oldReleaseFiles, oldGbdbSet, atticSet, revokedFiles, revokedTableSet, revokedGbdbs, missingFiles, newSupplementalSet, oldSupplementalSet, pushTables, pushFiles, pushGbdbs, newSupp) = (self.totalFiles, self.newGbdbSet, self.newTableSet, self.additionalList, self.oldAdditionalList, self.oldTableSet, self.oldTotalFiles, self.oldGbdbSet, self.atticSet, self.revokedFiles, self.revokedTableSet, self.revokedGbdbs, self.missingFiles, self.newSupplementalSet, self.oldSupplementalSet, self.pushTables, self.pushFiles, self.pushGbdbs, self.newSupp)
#the groups here need to be predefined, I just copied and pasted after working out what they were
sep = "\n"
output = []
#maths
allTables = newTableSet | oldTableSet | revokedTableSet
untouchedTables = oldTableSet & newTableSet
allFiles = totalFiles | oldReleaseFiles | revokedFiles
newFiles = pushFiles - revokedFiles
untouchedFiles = (totalFiles & oldReleaseFiles) - revokedFiles
filesNoRevoke = totalFiles - revokedFiles
allGbdbs = newGbdbSet | oldGbdbSet | revokedGbdbs
untouchedGbdbs = (newGbdbSet & oldGbdbSet) - revokedGbdbs
allSupp = newSupplementalSet | oldSupplementalSet
removedSupp = oldSupplementalSet - newSupplementalSet
untouchedSupp = oldSupplementalSet & newSupplementalSet
allOther = additionalList | oldAdditionalList
removedOther = oldAdditionalList - additionalList
output.extend(self.__qaHeader(output, newTableSet, filesNoRevoke, newGbdbSet, newSupp, oldSupplementalSet, additionalList, revokedTableSet, revokedFiles, revokedGbdbs, pushFiles, pushGbdbs, args, c))
output.extend(self.__printSection(pushTables, untouchedTables, revokedTableSet, allTables, "tables", 0, args['summary']))
output.extend(self.__printSection(pushFiles, untouchedFiles, revokedFiles, allFiles, "download", self.releasePath, args['summary']))
output.extend(self.__printSection(pushGbdbs, untouchedGbdbs, revokedGbdbs, allGbdbs, "gbdbs", self.gbdbPath, args['summary']))
output.extend(self.__printSection(newSupp, untouchedSupp, removedSupp, allSupp, "supplemental", self.releasePath, args['summary']))
#These attributes are the critical ones that are used by qaInit, others could potentially use these also.
otherprint = len(allOther)
if otherprint:
output.append("\n")
output.append("OTHER FILES:")
output.append("New: %s" % len(additionalList))
output.append("Revoked/Replace: %s" % len(removedOther))
output.append("Total: %s" % len(allOther))
if otherprint and not args['summary']:
output.append("")
output.append("New Other Files (%s):" % len(additionalList))
output.extend(sorted(list(self.newOthers)))
output.append("")
output.append("Revoked Other Files (%s):" % len(removedOther))
output.extend(ucscUtils.printIter((removedOther), self.releasePath))
output.append("\n")
output.extend(self.__addMissingToReport(missingFiles, "Files", self.releasePathOld))
output.append("\n")
output.extend(self.__addMissingToReport(self.droppedTables, "Tables"))
output.append("\n")
if self.atticSet:
if self.newInAttic:
output.append("New Attic Objects (%s):" % len(self.newInAttic))
output.extend(ucscUtils.printIter((self.newInAttic)))
output.append("\n")
if self.stillInAttic:
output.append("Untouched Attic Objects (%s):" % len(self.stillInAttic))
output.extend(ucscUtils.printIter((self.stillInAttic)))
output.append("\n")
if self.noMoreAttic:
output.append("Removed from Attic Objects (%s):" % len(self.noMoreAttic))
output.extend(ucscUtils.printIter((self.noMoreAttic)))
output.append("\n")
output.append("\n")
if not args['ignore']:
output.append("No Errors")
else:
output.append("The counts here were generated by ignoring errors, they may not be correct")
return output
def __init__(self, args):
self.releaseNew = args['releaseNew']
self.releaseOld = args['releaseOld']
self.database = args['database']
self.composite = args['composite']
self.loose = args['loose']
self.ignore = args['ignore']
self.summary = args['summary']
self.specialMdb = args['specialMdb']
self.args = args
if 'verbose' in args:
self.verbose = args['verbose']
else:
self.verbose = 0
errors = []
c = track.CompositeTrack(self.database, self.composite, None,
self.specialMdb)
#sanitize arguments
if not self.releaseOld.isdigit():
self.releaseOld = 'solo'
elif int(self.releaseOld) <= 0:
self.releaseOlf = 'solo'
elif self.releaseOld > self.releaseNew:
self.releaseOld = 'solo'
if self.verbose >= 1:
sys.stderr.write("Initializing MkChangeNotes\n")
self.releasePath = c.httpDownloadsPath + 'release' + args['releaseNew']
self.gbdbPath = "/gbdb/%s/bbi" % args['database']
self.trackDbFile = c.currentTrackDb
if not self.trackDbFile:
self.trackDb = None
errors.append(
"track: There is no entry in trackDb.wgEncode.ra for %s with the alpha tag"
% self.composite)
else:
self.trackDb = RaFile(self.trackDbFile, "track")
if int(self.releaseNew) > 1 and str(self.releaseOld) != 'solo':
if self.verbose >= 2:
sys.stderr.write("Comparison mode\n")
self.newReleaseFiles = c.releases[int(self.releaseNew) - 1]
self.oldReleaseFiles = c.releases[int(self.releaseOld) - 1]
self.releasePathOld = c.httpDownloadsPath + 'release' + args[
'releaseOld']
self.newMdb = c.alphaMetaDb
self.oldMdb = c.publicMetaDb
if self.verbose >= 2:
sys.stderr.write("Checking for missing files\n")
#make a list of missing files
self.missingFiles = self.__checkFilesForDropped()
#filter them out of old release files
if self.verbose >= 1:
sys.stderr.write("Scanning and parsing release directories\n")
#check if all files listed in release directories have associated metaDb entries
(self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet,
self.newSupplementalSet,
newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new")
(self.oldMdb, self.oldRevokedSet, self.oldRevokedFiles,
self.oldAtticSet, self.oldSupplementalSet,
oldFileErrors) = self.checkMetaDbForFiles("public metaDb", "old")
self.expIds = set(
self.newMdb.filter(lambda s: 'expId' in s,
lambda s: s['expId']))
if self.verbose >= 2:
sys.stderr.write("Checking for attic files\n")
#check that attic fiels aren't in trackDb
if self.trackDb:
errors.extend(self.__checkAtticNotInTrackDb())
#checks to see that nothing has disappeared between public and alpha
if self.verbose >= 1:
sys.stderr.write("Checking new metaDb for missing stanzas\n")
errors.extend(self.__checkAlphaForDropped("alpha metaDb",
"stanza"))
if self.verbose >= 1:
sys.stderr.write("Checking file md5sums across releases\n")
errors.extend(self.__checkMd5sums())
#checks and gets tables that are present, also returns a revoked set of tables for new
if self.verbose >= 1:
sys.stderr.write("Checking table status\n")
(self.newTableSet, self.revokedTableSet, self.newMissingTables,
newTableError) = self.checkTableStatus("alpha metaDb", "new")
(self.oldTableSet, spam, self.droppedTables,
oldTableError) = self.checkTableStatus("public metaDb", "old")
self.newInAttic = self.atticSet - self.oldAtticSet
self.stillInAttic = self.oldAtticSet & self.atticSet
self.oldTableSet = self.oldTableSet - self.atticSet
self.noMoreAttic = self.oldAtticSet - self.atticSet
self.changedTables = self.oldTableSet - self.newTableSet - self.revokedTableSet
#same as above except for gbdbs
if self.verbose >= 1:
sys.stderr.write("Checking GBDB status\n")
(self.newGbdbSet, self.revokedGbdbs,
newGbdbError) = self.getGbdbFiles("new")
(self.oldGbdbSet, eggs, oldGbdbError) = self.getGbdbFiles("old")
#remove missing files from gbdbs
self.oldGbdbSet = self.oldGbdbSet - self.missingFiles
self.oldGbdbSet = self.oldGbdbSet - self.atticSet
self.changedGbdbs = self.oldGbdbSet - self.newGbdbSet - self.revokedGbdbs
for i in self.missingFiles:
if i in self.oldReleaseFiles:
del self.oldReleaseFiles[i]
#fill in the errors
errors.extend(newFileErrors)
errors.extend(oldFileErrors)
errors.extend(newTableError)
errors.extend(oldTableError)
errors.extend(newGbdbError)
errors.extend(oldGbdbError)
if self.changedTables:
errors.append(
"These tables were tables in the old release, but are no longer tables in the new release:"
)
errors.extend(list(self.changedTables))
if self.changedGbdbs:
errors.append(
"These GBDBs were GBDB tables in the old release, but are no longer GBDB tables in the new release:"
)
errors.extend(list(self.changedGbdbs))
#for ease of typing
totalFiles = set(self.newReleaseFiles)
oldTotalFiles = set(self.oldReleaseFiles)
#these could honestly be moved earlier, get a file list processing section or something
#they clean out special fiels out and separated the master fiels list into the 3 required
#ones: wgEncode, supplemental and additional.
self.totalFiles = self.__cleanSpecialFiles(totalFiles)
self.oldTotalFiles = self.__cleanSpecialFiles(oldTotalFiles)
(self.oldTotalFiles, self.additionalList, self.oldAdditionalList,
self.totalFiles) = self.__separateOutAdditional()
#get the stuff you need to push
self.pushTables = set(sorted(
(self.newTableSet - self.oldTableSet)))
self.pushFiles = set(sorted(
(self.totalFiles - self.oldTotalFiles)))
self.pushGbdbs = set(sorted((self.newGbdbSet - self.oldGbdbSet)))
self.newSupp = self.newSupplementalSet - self.oldSupplementalSet
self.newTables = set(self.pushTables)
self.newFiles = set(
ucscUtils.printIter(self.pushFiles, self.releasePath))
self.newGbdbs = set(
ucscUtils.printIter(self.pushGbdbs, self.gbdbPath))
self.newSupplemental = set(
ucscUtils.printIter(self.newSupp, self.releasePath))
self.newOthers = set(
ucscUtils.printIter(self.additionalList, self.releasePath))
self.fullFiles = sorted(self.totalFiles - self.revokedFiles)
self.fullTables = self.oldTableSet & self.newTableSet
self.errors = errors
#don't output.append(report unless ignore option is on or no errors
#module mode doesn't generate output by default
if self.verbose >= 1:
sys.stderr.write("Creating report\n")
if (not errors) or self.ignore:
self.output = self.printReport(args, c)
else:
self.output = self.printErrors(errors, self.missingFiles)
elif self.releaseOld == 'solo':
self.newReleaseFiles = c.releases[int(self.releaseNew) - 1]
self.oldReleaseFiles = set()
self.newMdb = c.alphaMetaDb
#check that attic fiels aren't in trackDb
if self.trackDb:
errors.extend(self.__checkAtticNotInTrackDb())
(self.newMdb, self.revokedSet, self.revokedFiles, self.atticSet,
self.newSupplementalSet,
newFileErrors) = self.checkMetaDbForFiles("alpha metaDb", "new")
self.expIds = set(
self.newMdb.filter(lambda s: 'expId' in s,
lambda s: s['expId']))
(self.newTableSet, self.revokedTableSet, spam,
newTableError) = self.checkTableStatus("alpha metaDb", "new")
self.tableSize = self.__getTableSize()
(self.newGbdbSet, self.revokedGbdbs,
newGbdbError) = self.getGbdbFiles("new")
#collect errors
errors.extend(newFileErrors)
errors.extend(newTableError)
errors.extend(newGbdbError)
#set for easy operations
totalFiles = set(self.newReleaseFiles)
#clean out special fiels we don't push i.e. md5sum.history
self.totalFiles = self.__cleanSpecialFiles(totalFiles)
self.pushTables = self.newTableSet
self.pushFiles = self.totalFiles
self.pushGbdbs = self.newGbdbSet
self.newSupp = self.newSupplementalSet
self.fullFiles = self.totalFiles
self.fullTables = self.newTableSet
#makes list for additional files
(self.oldTotalFiles, self.oldSupplementalSet) = (set(), set())
(self.oldReleaseFiles, self.additionalList, self.oldAdditionalList,
self.totalFiles) = self.__separateOutAdditional()
self.errors = errors
if (not errors) or self.ignore:
self.output = self.printReportOne(args, c)
else:
self.droppedTables = set()
self.output = self.printErrors(errors, set())
def printReportOne(self, args, c):
(totalFiles, revokedFiles, newGbdbSet, revokedGbdbs, newTableSet,
revokedTables, additionalList, atticSet, newSupplementalSet,
tableSize) = (self.totalFiles, self.revokedFiles, self.newGbdbSet,
self.revokedGbdbs, self.newTableSet,
self.revokedTableSet, self.additionalList,
self.atticSet, self.newSupplementalSet, self.tableSize)
output = []
newTables = newTableSet - revokedTables
newFiles = totalFiles - revokedFiles
newGbdbs = newGbdbSet - revokedGbdbs
output.extend(
self.__qaHeader(output, newTables, newFiles, newGbdbSet,
newSupplementalSet, additionalList, revokedTables,
revokedFiles, revokedGbdbs, totalFiles, newGbdbSet,
args, c))
self.newTables = set(newTables)
self.newFiles = set(ucscUtils.printIter(newFiles, self.releasePath))
self.newGbdbs = set(ucscUtils.printIter(newGbdbs, self.gbdbPath))
self.newSupplemental = set(
ucscUtils.printIter(newSupplementalSet, self.releasePath))
self.newOthers = set(
ucscUtils.printIter(additionalList, self.releasePath))
if not args['summary']:
output.append("")
output.extend(
self.__printSectionOne(output, self.newTables, "New Tables"))
output.extend(
self.__printSectionOne(output, self.newFiles,
"New Download Files"))
output.extend(
self.__printSectionOne(output, self.newGbdbs,
"New Gbdb Files"))
output.extend(
self.__printSectionOne(output, self.newSupplemental,
"New Supplemental Files"))
output.extend(
self.__printSectionOne(output, self.newOthers,
"New Other Files"))
output.extend(
self.__printSectionOne(output,
ucscUtils.printIter(revokedTables, 0),
"Revoked Tables"))
output.extend(
self.__printSectionOne(
output, ucscUtils.printIter(revokedFiles,
self.releasePath),
"Revoked Files"))
output.extend(
self.__printSectionOne(
output, ucscUtils.printIter(revokedGbdbs, self.gbdbPath),
"Revoked Gbdbs"))
if self.atticSet:
output.append("Attic Objects")
output.extend(
ucscUtils.printIter((self.atticSet), self.releasePath))
if not args['ignore']:
output.append("No Errors")
else:
output.append(
"The counts here were generated by ignoring errors, they may not be correct"
)
return output
def printReport(self, args, c):
(totalFiles, newGbdbSet, newTableSet, additionalList,
oldAdditionalList, oldTableSet, oldReleaseFiles, oldGbdbSet, atticSet,
revokedFiles, revokedTableSet, revokedGbdbs, missingFiles,
newSupplementalSet, oldSupplementalSet, pushTables, pushFiles,
pushGbdbs,
newSupp) = (self.totalFiles, self.newGbdbSet, self.newTableSet,
self.additionalList, self.oldAdditionalList,
self.oldTableSet, self.oldTotalFiles, self.oldGbdbSet,
self.atticSet, self.revokedFiles, self.revokedTableSet,
self.revokedGbdbs, self.missingFiles,
self.newSupplementalSet, self.oldSupplementalSet,
self.pushTables, self.pushFiles, self.pushGbdbs,
self.newSupp)
#the groups here need to be predefined, I just copied and pasted after working out what they were
sep = "\n"
output = []
#maths
allTables = newTableSet | oldTableSet | revokedTableSet
untouchedTables = oldTableSet & newTableSet
allFiles = totalFiles | oldReleaseFiles | revokedFiles
newFiles = pushFiles - revokedFiles
untouchedFiles = (totalFiles & oldReleaseFiles) - revokedFiles
filesNoRevoke = totalFiles - revokedFiles
allGbdbs = newGbdbSet | oldGbdbSet | revokedGbdbs
untouchedGbdbs = (newGbdbSet & oldGbdbSet) - revokedGbdbs
allSupp = newSupplementalSet | oldSupplementalSet
removedSupp = oldSupplementalSet - newSupplementalSet
untouchedSupp = oldSupplementalSet & newSupplementalSet
allOther = additionalList | oldAdditionalList
removedOther = oldAdditionalList - additionalList
output.extend(
self.__qaHeader(output, newTableSet, filesNoRevoke, newGbdbSet,
newSupp, additionalList, revokedTableSet,
revokedFiles, revokedGbdbs, pushFiles, pushGbdbs,
args, c))
output.extend(
self.__printSection(pushTables, untouchedTables, revokedTableSet,
allTables, "tables", 0, args['summary']))
output.extend(
self.__printSection(pushFiles, untouchedFiles, revokedFiles,
allFiles, "download", self.releasePath,
args['summary']))
output.extend(
self.__printSection(pushGbdbs, untouchedGbdbs, revokedGbdbs,
allGbdbs, "gbdbs", self.gbdbPath,
args['summary']))
output.extend(
self.__printSection(newSupp, untouchedSupp, removedSupp, allSupp,
"supplemental", self.releasePath,
args['summary']))
#These attributes are the critical ones that are used by qaInit, others could potentially use these also.
otherprint = len(allOther)
if otherprint:
output.append("\n")
output.append("OTHER FILES:")
output.append("New: %s" % len(additionalList))
output.append("Revoked/Replace: %s" % len(removedOther))
output.append("Total: %s" % len(allOther))
if otherprint and not args['summary']:
output.append("")
output.append("New Other Files (%s):" % len(additionalList))
output.extend(sorted(list(self.newOthers)))
output.append("")
output.append("Revoked Other Files (%s):" % len(removedOther))
output.extend(ucscUtils.printIter((removedOther),
self.releasePath))
output.append("\n")
output.extend(
self.__addMissingToReport(missingFiles, "Files",
self.releasePathOld))
output.append("\n")
output.extend(self.__addMissingToReport(self.droppedTables, "Tables"))
output.append("\n")
if self.atticSet:
if self.newInAttic:
output.append("New Attic Objects (%s):" % len(self.newInAttic))
output.extend(ucscUtils.printIter((self.newInAttic)))
if self.stillInAttic:
output.append("Untouched Attic Objects (%s):" %
len(self.stillInAttic))
output.extend(ucscUtils.printIter((self.stillInAttic)))
if self.noMoreAttic:
output.append("Untouched Attic Objects (%s):" %
len(self.noMoreAttic))
output.extend(ucscUtils.printIter((self.noMoreAttic)))
output.append("\n")
if not args['ignore']:
output.append("No Errors")
else:
output.append(
"The counts here were generated by ignoring errors, they may not be correct"
)
return output