def saveStateRaw(stream, dataBlocks, stripMetadata = False):
writer = StringBuffer()
for block in dataBlocks:
writer.write('[%s#%s]\n' % (block[DataProvider.Dataset], block[DataProvider.BlockName]))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.DatasetID in block:
writer.write('id = %d\n' % block[DataProvider.DatasetID])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) != None:
writer.write('se list = %s\n' % str.join(',', block[DataProvider.Locations]))
cPrefix = os.path.commonprefix(map(lambda x: x[DataProvider.URL], block[DataProvider.FileList]))
cPrefix = str.join('/', cPrefix.split('/')[:-1])
if len(cPrefix) > 6:
writer.write('prefix = %s\n' % cPrefix)
formatter = lambda x: x.replace(cPrefix + '/', '')
else:
formatter = lambda x: x
writeMetadata = (DataProvider.Metadata in block) and not stripMetadata
if writeMetadata:
getMetadata = lambda fi, idxList: map(lambda idx: fi[DataProvider.Metadata][idx], idxList)
metadataHash = lambda fi, idx: utils.md5(repr(fi[DataProvider.Metadata][idx])).digest()
cMetadataIdx = range(len(block[DataProvider.Metadata]))
cMetadataHash = map(lambda idx: metadataHash(block[DataProvider.FileList][0], idx), cMetadataIdx)
for fi in block[DataProvider.FileList]: # Identify common metadata
for idx in filter(lambda idx: metadataHash(fi, idx) != cMetadataHash[idx], cMetadataIdx):
cMetadataIdx.remove(idx)
def filterC(common):
idxList = filter(lambda idx: (idx in cMetadataIdx) == common, range(len(block[DataProvider.Metadata])))
return utils.sorted(idxList, key = lambda idx: block[DataProvider.Metadata][idx])
writer.write('metadata = %s\n' % map(lambda idx: block[DataProvider.Metadata][idx], filterC(True) + filterC(False)))
if cMetadataIdx:
writer.write('metadata common = %s\n' % getMetadata(block[DataProvider.FileList][0], filterC(True)))
writeMetadata = len(cMetadataIdx) != len(block[DataProvider.Metadata])
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (formatter(fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if writeMetadata:
writer.write(' %s' % getMetadata(fi, filterC(False)))
writer.write('\n')
writer.write('\n')
stream.write(writer.getvalue())
def getHash(self): return utils.md5(str(self.values) + str(self.keys)).hexdigest()
def getHash(self): return utils.md5(str(self.key) + self.matcher.getHash() + self.psource.getHash()).hexdigest()
def getHash(self): return utils.md5(str(map(lambda x: self.lookupDict, self.lookupOrder))).hexdigest()
def getHash(self): return utils.md5(str(self.srcName) + str(self.dataSplitter.getMaxJobs())).hexdigest()