def saveStateRaw(stream, dataBlocks, stripMetadata = False):
writer = StringBuffer()
for block in dataBlocks:
writer.write('[%s#%s]\n' % (block[DataProvider.Dataset], block[DataProvider.BlockName]))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.DatasetID in block:
writer.write('id = %d\n' % block[DataProvider.DatasetID])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) != None:
writer.write('se list = %s\n' % str.join(',', block[DataProvider.Locations]))
cPrefix = os.path.commonprefix(map(lambda x: x[DataProvider.URL], block[DataProvider.FileList]))
cPrefix = str.join('/', cPrefix.split('/')[:-1])
if len(cPrefix) > 6:
writer.write('prefix = %s\n' % cPrefix)
formatter = lambda x: x.replace(cPrefix + '/', '')
else:
formatter = lambda x: x
writeMetadata = (DataProvider.Metadata in block) and not stripMetadata
if writeMetadata:
getMetadata = lambda fi, idxList: map(lambda idx: fi[DataProvider.Metadata][idx], idxList)
metadataHash = lambda fi, idx: utils.md5(repr(fi[DataProvider.Metadata][idx])).digest()
cMetadataIdx = range(len(block[DataProvider.Metadata]))
cMetadataHash = map(lambda idx: metadataHash(block[DataProvider.FileList][0], idx), cMetadataIdx)
for fi in block[DataProvider.FileList]: # Identify common metadata
for idx in filter(lambda idx: metadataHash(fi, idx) != cMetadataHash[idx], cMetadataIdx):
cMetadataIdx.remove(idx)
def filterC(common):
idxList = filter(lambda idx: (idx in cMetadataIdx) == common, range(len(block[DataProvider.Metadata])))
return utils.sorted(idxList, key = lambda idx: block[DataProvider.Metadata][idx])
writer.write('metadata = %s\n' % map(lambda idx: block[DataProvider.Metadata][idx], filterC(True) + filterC(False)))
if cMetadataIdx:
writer.write('metadata common = %s\n' % getMetadata(block[DataProvider.FileList][0], filterC(True)))
writeMetadata = len(cMetadataIdx) != len(block[DataProvider.Metadata])
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (formatter(fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if writeMetadata:
writer.write(' %s' % getMetadata(fi, filterC(False)))
writer.write('\n')
writer.write('\n')
stream.write(writer.getvalue())
def saveToStream(stream, dataBlocks, stripMetadata=False):
writer = StringBuffer()
for block in dataBlocks:
writer.write(
'[%s#%s]\n' %
(block[DataProvider.Dataset], block[DataProvider.BlockName]))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.DatasetID in block:
writer.write('id = %d\n' % block[DataProvider.DatasetID])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) is not None:
writer.write('se list = %s\n' %
str.join(',', block[DataProvider.Locations]))
cPrefix = os.path.commonprefix(
lmap(lambda x: x[DataProvider.URL],
block[DataProvider.FileList]))
cPrefix = str.join('/', cPrefix.split('/')[:-1])
if len(cPrefix) > 6:
writer.write('prefix = %s\n' % cPrefix)
formatter = lambda x: x.replace(cPrefix + '/', '')
else:
formatter = identity
writeMetadata = (DataProvider.Metadata
in block) and not stripMetadata
if writeMetadata:
(idxListBlock,
idxListFile) = DataProvider.classifyMetadataKeys(block)
def getMetadata(fi, idxList):
idxList = ifilter(
lambda idx: idx < len(fi[DataProvider.Metadata]),
idxList)
return json.dumps(
lmap(lambda idx: fi[DataProvider.Metadata][idx],
idxList))
writer.write('metadata = %s\n' % json.dumps(
lmap(lambda idx: block[DataProvider.Metadata][idx],
idxListBlock + idxListFile)))
if idxListBlock:
writer.write('metadata common = %s\n' % getMetadata(
block[DataProvider.FileList][0], idxListBlock))
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (formatter(
fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if writeMetadata and idxListFile:
writer.write(' %s' % getMetadata(fi, idxListFile))
writer.write('\n')
writer.write('\n')
stream.write(writer.getvalue())
def saveToStream(stream, dataBlocks, stripMetadata = False):
writer = StringBuffer()
for block in dataBlocks:
writer.write('[%s#%s]\n' % (block[DataProvider.Dataset], block[DataProvider.BlockName]))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.DatasetID in block:
writer.write('id = %d\n' % block[DataProvider.DatasetID])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) is not None:
writer.write('se list = %s\n' % str.join(',', block[DataProvider.Locations]))
cPrefix = os.path.commonprefix(lmap(lambda x: x[DataProvider.URL], block[DataProvider.FileList]))
cPrefix = str.join('/', cPrefix.split('/')[:-1])
if len(cPrefix) > 6:
writer.write('prefix = %s\n' % cPrefix)
formatter = lambda x: x.replace(cPrefix + '/', '')
else:
formatter = identity
writeMetadata = (DataProvider.Metadata in block) and not stripMetadata
if writeMetadata:
(idxListBlock, idxListFile) = DataProvider.classifyMetadataKeys(block)
def getMetadata(fi, idxList):
idxList = ifilter(lambda idx: idx < len(fi[DataProvider.Metadata]), idxList)
return json.dumps(lmap(lambda idx: fi[DataProvider.Metadata][idx], idxList))
writer.write('metadata = %s\n' % json.dumps(lmap(lambda idx: block[DataProvider.Metadata][idx], idxListBlock + idxListFile)))
if idxListBlock:
writer.write('metadata common = %s\n' % getMetadata(block[DataProvider.FileList][0], idxListBlock))
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (formatter(fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if writeMetadata and idxListFile:
writer.write(' %s' % getMetadata(fi, idxListFile))
writer.write('\n')
writer.write('\n')
stream.write(writer.getvalue())
def save_to_stream(stream, block_iter, strip_metadata=False):
writer = StringBuffer()
write_separator = False
for block in block_iter:
if write_separator:
writer.write('\n')
writer.write('[%s]\n' % DataProvider.get_block_id(block))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) is not None:
writer.write('se list = %s\n' %
str.join(',', block[DataProvider.Locations]))
common_prefix = os.path.commonprefix(
lmap(itemgetter(DataProvider.URL),
block[DataProvider.FileList]))
common_prefix = str.join('/', common_prefix.split('/')[:-1])
if len(common_prefix) > 6:
def _formatter(value):
return value.replace(common_prefix + '/', '')
writer.write('prefix = %s\n' % common_prefix)
else:
_formatter = identity
do_write_metadata = (DataProvider.Metadata
in block) and not strip_metadata
if do_write_metadata:
def _get_metadata_str(fi, idx_list):
idx_list = ifilter(
lambda idx: idx < len(fi[DataProvider.Metadata]),
idx_list)
return json.dumps(
lmap(lambda idx: fi[DataProvider.Metadata][idx],
idx_list))
(metadata_idx_list_block,
metadata_idx_list_file) = _split_metadata_idx_list(block)
metadata_header_str = json.dumps(
lmap(lambda idx: block[DataProvider.Metadata][idx],
metadata_idx_list_block + metadata_idx_list_file))
writer.write('metadata = %s\n' % metadata_header_str)
if metadata_idx_list_block:
metadata_str = _get_metadata_str(
block[DataProvider.FileList][0],
metadata_idx_list_block)
writer.write('metadata common = %s\n' % metadata_str)
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (_formatter(
fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if do_write_metadata and metadata_idx_list_file:
writer.write(' %s' %
_get_metadata_str(fi, metadata_idx_list_file))
writer.write('\n')
stream.write(writer.getvalue())
erase_content(writer)
write_separator = True
yield block
writer.close()