def _diffParams(self, psource_old, psource_new, mapJob2PID, redoNewPNum, disableNewPNum):
# Reduces psource output to essential information for diff - faster than keying
def translatePSource(psource):
keys_store = sorted(ifilter(lambda k: not k.untracked, psource.getJobKeys()))
def translateEntry(meta): # Translates parameter setting into hash
tmp = md5()
for key in ifilter(lambda k: k in meta, keys_store):
if str(meta[key]):
tmp.update(str2bytes(key))
tmp.update(str2bytes(str(meta[key])))
return { ParameterInfo.HASH: tmp.hexdigest(), 'GC_PARAM': meta['GC_PARAM'],
ParameterInfo.ACTIVE: meta[ParameterInfo.ACTIVE] }
for entry in psource.iterJobs():
yield translateEntry(entry)
params_old = list(translatePSource(psource_old))
params_new = list(translatePSource(psource_new))
def sameParams(paramsAdded, paramsMissing, paramsSame, oldParam, newParam):
mapJob2PID[oldParam['GC_PARAM']] = newParam['GC_PARAM']
if not oldParam[ParameterInfo.ACTIVE] and newParam[ParameterInfo.ACTIVE]:
redoNewPNum.add(newParam['GC_PARAM'])
if oldParam[ParameterInfo.ACTIVE] and not newParam[ParameterInfo.ACTIVE]:
disableNewPNum.add(newParam['GC_PARAM'])
return utils.DiffLists(params_old, params_new, itemgetter(ParameterInfo.HASH), sameParams)
def resyncSources(oldBlocks, newBlocks): # Compare different blocks according to their name - NOT full content def keyBlock(x): return (x[DataProvider.Dataset], x[DataProvider.BlockName]) sort_inplace(oldBlocks, key = keyBlock) sort_inplace(newBlocks, key = keyBlock) def onMatchingBlock(blocksAdded, blocksMissing, blocksMatching, oldBlock, newBlock): # Compare different files according to their name - NOT full content def keyFiles(x): return x[DataProvider.URL] sort_inplace(oldBlock[DataProvider.FileList], key = keyFiles) sort_inplace(newBlock[DataProvider.FileList], key = keyFiles) def onMatchingFile(filesAdded, filesMissing, filesMatched, oldFile, newFile): filesMatched.append((oldFile, newFile)) (filesAdded, filesMissing, filesMatched) = \ utils.DiffLists(oldBlock[DataProvider.FileList], newBlock[DataProvider.FileList], keyFiles, onMatchingFile, isSorted = True) if filesAdded: # Create new block for added files in an existing block tmpBlock = copy.copy(newBlock) tmpBlock[DataProvider.FileList] = filesAdded tmpBlock[DataProvider.NEntries] = sum(imap(lambda x: x[DataProvider.NEntries], filesAdded)) blocksAdded.append(tmpBlock) blocksMatching.append((oldBlock, newBlock, filesMissing, filesMatched)) return utils.DiffLists(oldBlocks, newBlocks, keyBlock, onMatchingBlock, isSorted = True)