def show_config(self): buffer = StringBuffer() try: self._config.write(buffer) return _tag('pre', _tag('code', buffer.getvalue())) finally: buffer.close()
def _get_dataset_hash(self): buffer = StringBuffer() for _ in DataProvider.save_to_stream(buffer, self.iter_blocks_normed()): pass value = buffer.getvalue() buffer.close() return md5_hex(value)
def save_to_stream(stream, block_iter, strip_metadata=False): writer = StringBuffer() write_separator = False for block in block_iter: if write_separator: writer.write('\n') writer.write('[%s]\n' % DataProvider.get_block_id(block)) if DataProvider.Nickname in block: writer.write('nickname = %s\n' % block[DataProvider.Nickname]) if DataProvider.NEntries in block: writer.write('events = %d\n' % block[DataProvider.NEntries]) if block.get(DataProvider.Locations) is not None: writer.write('se list = %s\n' % str.join(',', block[DataProvider.Locations])) common_prefix = os.path.commonprefix( lmap(itemgetter(DataProvider.URL), block[DataProvider.FileList])) common_prefix = str.join('/', common_prefix.split('/')[:-1]) if len(common_prefix) > 6: def _formatter(value): return value.replace(common_prefix + '/', '') writer.write('prefix = %s\n' % common_prefix) else: _formatter = identity do_write_metadata = (DataProvider.Metadata in block) and not strip_metadata if do_write_metadata: def _get_metadata_str(fi, idx_list): idx_list = ifilter( lambda idx: idx < len(fi[DataProvider.Metadata]), idx_list) return json.dumps( lmap(lambda idx: fi[DataProvider.Metadata][idx], idx_list)) (metadata_idx_list_block, metadata_idx_list_file) = _split_metadata_idx_list(block) metadata_header_str = json.dumps( lmap(lambda idx: block[DataProvider.Metadata][idx], metadata_idx_list_block + metadata_idx_list_file)) writer.write('metadata = %s\n' % metadata_header_str) if metadata_idx_list_block: metadata_str = _get_metadata_str( block[DataProvider.FileList][0], metadata_idx_list_block) writer.write('metadata common = %s\n' % metadata_str) for fi in block[DataProvider.FileList]: writer.write('%s = %d' % (_formatter( fi[DataProvider.URL]), fi[DataProvider.NEntries])) if do_write_metadata and metadata_idx_list_file: writer.write(' %s' % _get_metadata_str(fi, metadata_idx_list_file)) writer.write('\n') stream.write(writer.getvalue()) erase_content(writer) write_separator = True yield block writer.close()