def dataset_show_diff(options):
if len(options.args) != 2:
options.parser.exit_with_usage(options.parser.usage('data'))
provider_a = DataProvider.load_from_file(options.args[0])
provider_b = DataProvider.load_from_file(options.args[1])
block_resync_tuple = DataProvider.resync_blocks(provider_a.get_block_list_cached(show_stats=False),
provider_b.get_block_list_cached(show_stats=False))
(block_list_added, block_list_missing, block_list_matching) = block_resync_tuple
def _dataset_iter_matching_blocks():
for (block_old, block_new, _, _) in block_list_matching:
def _format_change(old, new):
if old != new:
return '%s -> %s' % (old, new)
return old
block_old[DataProvider.NFiles] = _format_change(len(block_old.get(DataProvider.FileList, [])),
len(block_new.get(DataProvider.FileList, [])))
block_old[DataProvider.NEntries] = _format_change(block_old[DataProvider.NEntries],
block_new[DataProvider.NEntries])
yield block_old
header_list = [(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'),
(DataProvider.NFiles, '#Files'), (DataProvider.NEntries, '#Entries')]
if block_list_added:
ConsoleTable.create(header_list, dataset_iter_blocks(block_list_added), title='Added blocks')
if block_list_missing:
ConsoleTable.create(header_list, dataset_iter_blocks(block_list_missing), title='Removed blocks')
if block_list_matching:
ConsoleTable.create(header_list, _dataset_iter_matching_blocks(), title='Matching blocks')
def dataset_show_diff(options):
if len(options.args) != 2:
options.parser.exit_with_usage(options.parser.usage('data'))
provider_a = DataProvider.load_from_file(options.args[0])
provider_b = DataProvider.load_from_file(options.args[1])
block_resync_tuple = DataProvider.resync_blocks(
provider_a.get_block_list_cached(show_stats=False),
provider_b.get_block_list_cached(show_stats=False))
(block_list_added, block_list_missing,
block_list_matching) = block_resync_tuple
def _dataset_iter_matching_blocks():
for (block_old, block_new, _, _) in block_list_matching:
def _format_change(old, new):
if old != new:
return '%s -> %s' % (old, new)
return old
block_old[DataProvider.NFiles] = _format_change(
len(block_old.get(DataProvider.FileList, [])),
len(block_new.get(DataProvider.FileList, [])))
block_old[DataProvider.NEntries] = _format_change(
block_old[DataProvider.NEntries],
block_new[DataProvider.NEntries])
yield block_old
header_list = [(DataProvider.Dataset, 'Dataset'),
(DataProvider.BlockName, 'Block'),
(DataProvider.NFiles, '#Files'),
(DataProvider.NEntries, '#Entries')]
if block_list_added:
ConsoleTable.create(header_list,
dataset_iter_blocks(block_list_added),
title='Added blocks')
if block_list_missing:
ConsoleTable.create(header_list,
dataset_iter_blocks(block_list_missing),
title='Removed blocks')
if block_list_matching:
ConsoleTable.create(header_list,
_dataset_iter_matching_blocks(),
title='Matching blocks')
def dataset_show_removed(options):
if len(options.args) < 2:
options.parser.exit_with_usage(options.parser.usage('data'))
block_list_missing = []
provider_old = DataProvider.load_from_file(options.args[0])
for dataset_fn in options.args[1:]:
provider_new = DataProvider.load_from_file(dataset_fn)
block_resync_tuple = DataProvider.resync_blocks(
provider_old.get_block_list_cached(show_stats=False),
provider_new.get_block_list_cached(show_stats=False))
for block in block_resync_tuple[1]: # iterate missing block list
tmp = dict(block)
tmp[DataProvider.RemovedIn] = dataset_fn
block_list_missing.append(tmp)
provider_old = provider_new
if block_list_missing:
ConsoleTable.create([(DataProvider.Dataset, 'Dataset'), (DataProvider.BlockName, 'Block'),
(DataProvider.NFiles, '#Files'), (DataProvider.NEntries, '#Entries'),
(DataProvider.RemovedIn, 'Removed in file')],
dataset_iter_blocks(block_list_missing), title='Removed blocks')
def dataset_show_removed(options):
if len(options.args) < 2:
options.parser.exit_with_usage(options.parser.usage('data'))
block_list_missing = []
provider_old = DataProvider.load_from_file(options.args[0])
for dataset_fn in options.args[1:]:
provider_new = DataProvider.load_from_file(dataset_fn)
block_resync_tuple = DataProvider.resync_blocks(
provider_old.get_block_list_cached(show_stats=False),
provider_new.get_block_list_cached(show_stats=False))
for block in block_resync_tuple[1]: # iterate missing block list
tmp = dict(block)
tmp[DataProvider.RemovedIn] = dataset_fn
block_list_missing.append(tmp)
provider_old = provider_new
if block_list_missing:
ConsoleTable.create([(DataProvider.Dataset, 'Dataset'),
(DataProvider.BlockName, 'Block'),
(DataProvider.NFiles, '#Files'),
(DataProvider.NEntries, '#Entries'),
(DataProvider.RemovedIn, 'Removed in file')],
dataset_iter_blocks(block_list_missing),
title='Removed blocks')
