def show_config(self):
buffer = StringBuffer()
try:
self._config.write(buffer)
return _tag('pre', _tag('code', buffer.getvalue()))
finally:
buffer.close()
def _get_dataset_hash(self):
buffer = StringBuffer()
for _ in DataProvider.save_to_stream(buffer,
self.iter_blocks_normed()):
pass
value = buffer.getvalue()
buffer.close()
return md5_hex(value)
def save_to_stream(stream, block_iter, strip_metadata=False):
writer = StringBuffer()
write_separator = False
for block in block_iter:
if write_separator:
writer.write('\n')
writer.write('[%s]\n' % DataProvider.get_block_id(block))
if DataProvider.Nickname in block:
writer.write('nickname = %s\n' % block[DataProvider.Nickname])
if DataProvider.NEntries in block:
writer.write('events = %d\n' % block[DataProvider.NEntries])
if block.get(DataProvider.Locations) is not None:
writer.write('se list = %s\n' %
str.join(',', block[DataProvider.Locations]))
common_prefix = os.path.commonprefix(
lmap(itemgetter(DataProvider.URL),
block[DataProvider.FileList]))
common_prefix = str.join('/', common_prefix.split('/')[:-1])
if len(common_prefix) > 6:
def _formatter(value):
return value.replace(common_prefix + '/', '')
writer.write('prefix = %s\n' % common_prefix)
else:
_formatter = identity
do_write_metadata = (DataProvider.Metadata
in block) and not strip_metadata
if do_write_metadata:
def _get_metadata_str(fi, idx_list):
idx_list = ifilter(
lambda idx: idx < len(fi[DataProvider.Metadata]),
idx_list)
return json.dumps(
lmap(lambda idx: fi[DataProvider.Metadata][idx],
idx_list))
(metadata_idx_list_block,
metadata_idx_list_file) = _split_metadata_idx_list(block)
metadata_header_str = json.dumps(
lmap(lambda idx: block[DataProvider.Metadata][idx],
metadata_idx_list_block + metadata_idx_list_file))
writer.write('metadata = %s\n' % metadata_header_str)
if metadata_idx_list_block:
metadata_str = _get_metadata_str(
block[DataProvider.FileList][0],
metadata_idx_list_block)
writer.write('metadata common = %s\n' % metadata_str)
for fi in block[DataProvider.FileList]:
writer.write('%s = %d' % (_formatter(
fi[DataProvider.URL]), fi[DataProvider.NEntries]))
if do_write_metadata and metadata_idx_list_file:
writer.write(' %s' %
_get_metadata_str(fi, metadata_idx_list_file))
writer.write('\n')
stream.write(writer.getvalue())
erase_content(writer)
write_separator = True
yield block
writer.close()