def test_nested_run(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(identity, 42) task = runforrest.defer(identity, task) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=4)) assert tasks[0].returnvalue == 42
def test_task_indexing(): tasklist = runforrest.TaskList('tmp', post_clean=True) # send something that can be indexed: task = runforrest.defer(identity, [42]) # retrieve at index: task = runforrest.defer(identity, task[0]) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=1)) assert tasks[0].returnvalue == 42
def test_task_accessor(): tasklist = runforrest.TaskList('tmp', post_clean=True) # send something that has an attribute: task = runforrest.defer(identity, Exception(42)) # retrieve the attribute: task = runforrest.defer(identity, task.args) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=1)) assert tasks[0].returnvalue == (42, )
def test_multiple_nested_runs(howmany=20): tasklist = runforrest.TaskList('tmp', post_clean=True) for v in range(howmany): task = runforrest.defer(identity, v) task = runforrest.defer(identity, task) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=10)) assert len(tasks) == howmany for r, v in zip(sorted(t.returnvalue for t in tasks), range(howmany)): assert r == v
def test_noschedule(): tasklist = runforrest.TaskList('tmp') task = runforrest.defer(identity, 42) tasklist.schedule(task) # do not run, but reopen: tasklist = runforrest.TaskList('tmp', noschedule_if_exist=True, post_clean=True) task = runforrest.defer(identity, 42) tasklist.schedule(task) # should now be a no-op tasks = list(tasklist.run(nprocesses=4)) assert len(tasks) == 1 assert tasks[0].returnvalue == 42
def test_metadata(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(identity, 42) tasklist.schedule(task, metadata='the truth') list(tasklist.run(nprocesses=1)) done = list(tasklist.done_tasks()) assert done[0].metadata == 'the truth'
def test_todo_and_done_task_access(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(identity, 42) tasklist.schedule(task) todo = list(tasklist.todo_tasks()) list(tasklist.run(nprocesses=1)) done = list(tasklist.done_tasks()) assert task == done[0] == todo[0]
def test_post_clean_true(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(crash) tasklist.schedule(task) list(tasklist.run(nprocesses=1)) del tasklist assert not pathlib.Path('tmp').exists() assert not pathlib.Path('tmp/todo').exists() assert not pathlib.Path('tmp/done').exists() assert not pathlib.Path('tmp/fail').exists()
def test_post_clean_false(): tasklist = runforrest.TaskList('tmp', post_clean=False) task = runforrest.defer(crash) tasklist.schedule(task) list(tasklist.run(nprocesses=1)) del tasklist for path in ['tmp/todo', 'tmp/done', 'tmp/fail', 'tmp']: path = pathlib.Path(path) assert path.exists() for file in path.iterdir(): file.unlink() path.rmdir()
def test_invalid_task(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(0) # not a function! tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=1)) assert len(tasks) == 1 fail = list(tasklist.fail_tasks()) assert len(fail) == 1 assert fail[0] == task assert isinstance(fail[0].errorvalue, TypeError) done = list(tasklist.done_tasks()) assert len(done) == 0
def test_failing_task(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(crash) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=1)) assert len(tasks) == 1 assert tasks[0] == task fail = list(tasklist.fail_tasks()) assert len(fail) == 1 assert fail[0] == task assert fail[0].errorvalue.args == ('TESTING', ) done = list(tasklist.done_tasks()) assert len(done) == 0
def test_logging(): logfile = pathlib.Path('tmp.log') if logfile.exists(): logfile.unlink() tasklist = runforrest.TaskList('tmp', post_clean=True, logfile=logfile) task = runforrest.defer(identity, 42) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=4)) with logfile.open() as f: lines = list(f) assert len(lines) == 3 assert 'schedule' in lines[0] assert 'start' in lines[1] assert 'done' in lines[2] logfile.unlink()
tasklist = TaskList('ground truth experiment', noschedule_if_exist=True, logfile='ground truth experiment.log') # calculate pitches for all algorithms and all sound files: for datasetname, dataset in dict(keele=KEELE, keele_mod=KEELE_mod, ptdb_tug=PTDB_TUG, fda=FDA, timit=TIMIT, mocha_timit=MOCHA_TIMIT, cmu_arctic=CMU_Arctic).items(): for item in tqdm(dataset.all_items(), desc=f'preparing {datasetname}'): signal = defer(item).signal for algo in algos: task = defer(algo, signal, signal.metadata['samplerate']) tasklist.schedule(task, metadata=dict(item=item, dataset=datasetname, algo=algo)) for task in tqdm(tasklist.run(nprocesses=16, autokill=600), smoothing=0, desc='processing'): pass # works for all items and algos except YIN and TIMIT for SX136GCS0 and SI572DMT0 # collect all pitches into a new dataset: dataset = jbof.create_dataset('ground truth data')
def test_data_task(): tasklist = runforrest.TaskList('tmp', post_clean=True) task = runforrest.defer(42) tasklist.schedule(task) tasks = list(tasklist.run(nprocesses=4)) assert tasks[0].returnvalue == 42