def compute_items_todo(data_files,
t_str,
log_files,
must_redo=False,
sniff=1024,
no_sniffer=False):
if data_files and t_str:
todo = (_compute_data_ids(data_files, sniff, no_sniffer)
& int_ranges2set(t_str))
elif data_files:
todo = _compute_data_ids(data_files, sniff)
elif t_str:
todo = int_ranges2set(t_str)
else:
raise MissingSourceError('compute_todo')
completed = set()
failed = set()
if log_files:
for filename in log_files:
log_parser = LogParser()
events = log_parser.parse(filename)
for event in events:
if event.type == 'completed':
todo.discard(event.item_id)
completed.add(event.item_id)
elif event.type == 'failed':
failed.add(event.item_id)
if not must_redo:
failed -= completed
todo -= failed
return todo, completed, failed
def test_invalid_expr2set(self):
try:
_ = int_ranges2set('-1')
self.assertTrue(False)
except InvalidRangeSpecError:
pass
msg = ('### error: either both --empty and --reduce or neither '
'should be given\n')
sys.stderr.write(msg)
sys.exit(1)
try:
var_names = get_var_config(conf_filename)
if options.empty:
empty_filename, reduce_filename = options.empty, options.reduce
else:
mode = options.mode if options.mode else None
empty_script, reduce_script = get_mode_config(conf_filename, mode)
empty_filename = os.path.join(os.path.dirname(__file__), '..',
'reduce', empty_script)
reduce_filename = os.path.join(os.path.dirname(__file__), '..',
'reduce', reduce_script)
array_ids = list(int_ranges2set(options.t))
if not array_ids:
sys.stderr.write('empty array ID set\n')
sys.exit(0)
array_ids.sort()
array_id_idx = 0
while True:
array_id = array_ids[array_id_idx]
args = {
't': array_id,
var_names['array_idx_var']: array_id,
}
filename = options.pattern.format(**args)
if os.path.exists(filename):
cmd = [empty_filename, options.out, filename]
if options.empty_args:
def test_multiple_ind2set(self):
expected_set = set(range(5, 11)) | set(range(21, 31))
expected_set.add(3)
expected_set.add(15)
expected_set.add(35)
self.assertEqual(expected_set, int_ranges2set('3,5-10,15,35,21-30'))
def test_multiple2set(self):
expected_set = set(range(1, 11)) | set(range(21, 31))
self.assertEqual(expected_set, int_ranges2set('1-10,21-30'))
def test_simple2set(self):
expected_set = set(range(1, 11))
self.assertEqual(expected_set, int_ranges2set('1-10'))
def test_ind2set(self):
expected_set = set([2, 4, 8])
self.assertEqual(expected_set, int_ranges2set('2,4,8'))
def test_single2set(self):
expected_set = set([2])
self.assertEqual(expected_set, int_ranges2set('2'))