def saveStateRaw(stream, dataBlocks, stripMetadata = False): writer = StringBuffer() for block in dataBlocks: writer.write('[%s#%s]\n' % (block[DataProvider.Dataset], block[DataProvider.BlockName])) if DataProvider.Nickname in block: writer.write('nickname = %s\n' % block[DataProvider.Nickname]) if DataProvider.DatasetID in block: writer.write('id = %d\n' % block[DataProvider.DatasetID]) if DataProvider.NEntries in block: writer.write('events = %d\n' % block[DataProvider.NEntries]) if block.get(DataProvider.Locations) != None: writer.write('se list = %s\n' % str.join(',', block[DataProvider.Locations])) cPrefix = os.path.commonprefix(map(lambda x: x[DataProvider.URL], block[DataProvider.FileList])) cPrefix = str.join('/', cPrefix.split('/')[:-1]) if len(cPrefix) > 6: writer.write('prefix = %s\n' % cPrefix) formatter = lambda x: x.replace(cPrefix + '/', '') else: formatter = lambda x: x writeMetadata = (DataProvider.Metadata in block) and not stripMetadata if writeMetadata: getMetadata = lambda fi, idxList: map(lambda idx: fi[DataProvider.Metadata][idx], idxList) metadataHash = lambda fi, idx: utils.md5(repr(fi[DataProvider.Metadata][idx])).digest() cMetadataIdx = range(len(block[DataProvider.Metadata])) cMetadataHash = map(lambda idx: metadataHash(block[DataProvider.FileList][0], idx), cMetadataIdx) for fi in block[DataProvider.FileList]: # Identify common metadata for idx in filter(lambda idx: metadataHash(fi, idx) != cMetadataHash[idx], cMetadataIdx): cMetadataIdx.remove(idx) def filterC(common): idxList = filter(lambda idx: (idx in cMetadataIdx) == common, range(len(block[DataProvider.Metadata]))) return utils.sorted(idxList, key = lambda idx: block[DataProvider.Metadata][idx]) writer.write('metadata = %s\n' % map(lambda idx: block[DataProvider.Metadata][idx], filterC(True) + filterC(False))) if cMetadataIdx: writer.write('metadata common = %s\n' % getMetadata(block[DataProvider.FileList][0], filterC(True))) writeMetadata = len(cMetadataIdx) != len(block[DataProvider.Metadata]) for fi in block[DataProvider.FileList]: writer.write('%s = %d' % (formatter(fi[DataProvider.URL]), fi[DataProvider.NEntries])) if writeMetadata: writer.write(' %s' % getMetadata(fi, filterC(False))) writer.write('\n') writer.write('\n') stream.write(writer.getvalue())
def getHash(self): return utils.md5(str(self.values) + str(self.keys)).hexdigest()
def getHash(self): return utils.md5(str(self.key) + self.matcher.getHash() + self.psource.getHash()).hexdigest()
def getHash(self): return utils.md5(str(map(lambda x: self.lookupDict, self.lookupOrder))).hexdigest()
def getHash(self): return utils.md5(str(self.srcName) + str(self.dataSplitter.getMaxJobs())).hexdigest()