def dataset_show_diff(options): if len(options.args) != 2: options.parser.exit_with_usage(options.parser.usage('data')) provider_a = DataProvider.load_from_file(options.args[0]) provider_b = DataProvider.load_from_file(options.args[1]) block_resync_tuple = DataProvider.resync_blocks(provider_a.get_block_list_cached(show_stats=False), provider_b.get_block_list_cached(show_stats=False)) (block_list_added, block_list_missing, block_list_matching) = block_resync_tuple def _dataset_iter_matching_blocks(): for (block_old, block_new, _, _) in block_list_matching: def _format_change(old, new): if old != new: return '%s -> %s' % (old, new) return old block_old[DataProvider.NFiles] = _format_change(len(block_old.get(DataProvider.FileList, [])), len(block_new.get(DataProvider.FileList, []))) block_old[DataProvider.NEntries] = _format_change(block_old[DataProvider.NEntries], block_new[DataProvider.NEntries]) yield block_old header_list = [(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'), (DataProvider.NFiles, '#Files'), (DataProvider.NEntries, '#Entries')] if block_list_added: ConsoleTable.create(header_list, dataset_iter_blocks(block_list_added), title='Added blocks') if block_list_missing: ConsoleTable.create(header_list, dataset_iter_blocks(block_list_missing), title='Removed blocks') if block_list_matching: ConsoleTable.create(header_list, _dataset_iter_matching_blocks(), title='Matching blocks')
def dataset_show_diff(options): if len(options.args) != 2: options.parser.exit_with_usage(options.parser.usage('data')) provider_a = DataProvider.load_from_file(options.args[0]) provider_b = DataProvider.load_from_file(options.args[1]) block_resync_tuple = DataProvider.resync_blocks( provider_a.get_block_list_cached(show_stats=False), provider_b.get_block_list_cached(show_stats=False)) (block_list_added, block_list_missing, block_list_matching) = block_resync_tuple def _dataset_iter_matching_blocks(): for (block_old, block_new, _, _) in block_list_matching: def _format_change(old, new): if old != new: return '%s -> %s' % (old, new) return old block_old[DataProvider.NFiles] = _format_change( len(block_old.get(DataProvider.FileList, [])), len(block_new.get(DataProvider.FileList, []))) block_old[DataProvider.NEntries] = _format_change( block_old[DataProvider.NEntries], block_new[DataProvider.NEntries]) yield block_old header_list = [(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'), (DataProvider.NFiles, '#Files'), (DataProvider.NEntries, '#Entries')] if block_list_added: ConsoleTable.create(header_list, dataset_iter_blocks(block_list_added), title='Added blocks') if block_list_missing: ConsoleTable.create(header_list, dataset_iter_blocks(block_list_missing), title='Removed blocks') if block_list_matching: ConsoleTable.create(header_list, _dataset_iter_matching_blocks(), title='Matching blocks')
def dataset_show_removed(options): if len(options.args) < 2: options.parser.exit_with_usage(options.parser.usage('data')) block_list_missing = [] provider_old = DataProvider.load_from_file(options.args[0]) for dataset_fn in options.args[1:]: provider_new = DataProvider.load_from_file(dataset_fn) block_resync_tuple = DataProvider.resync_blocks( provider_old.get_block_list_cached(show_stats=False), provider_new.get_block_list_cached(show_stats=False)) for block in block_resync_tuple[1]: # iterate missing block list tmp = dict(block) tmp[DataProvider.RemovedIn] = dataset_fn block_list_missing.append(tmp) provider_old = provider_new if block_list_missing: ConsoleTable.create([(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'), (DataProvider.NFiles, '#Files'), (DataProvider.NEntries, '#Entries'), (DataProvider.RemovedIn, 'Removed in file')], dataset_iter_blocks(block_list_missing), title='Removed blocks')