def crtq(): tq = TaskQueue(FILE_QURL) tq.purge() tq.rezero() tq.insert((PrintTask(i) for i in range(N))) return tq
def test_renew(): tq = TaskQueue(FILE_QURL) tq.purge() tq.insert(PrintTask('hello')) ts = lambda fname: int(fname.split('--')[0]) ident = lambda fname: fname.split('--')[1] filenames = os.listdir(tq.api.queue_path) assert len(filenames) == 1 filename = filenames[0] assert ts(filename) == 0 identity = ident(filename) now = time.time() tq.renew(filename, 1) filenames = os.listdir(tq.api.queue_path) assert len(filenames) == 1 filename = filenames[0] assert ts(filename) >= int(time.time()) + 1 assert ident(filename) == identity
def test_single_threaded_insertion(): global QUEUE_NAME for qtype in QTYPES: tq = TaskQueue(n_threads=0, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL) tq.purge() if qtype != 'aws': assert tq.enqueued == 0 n_inserts = 5 for _ in range(n_inserts): task = PrintTask() tq.insert(task) tq.wait() if qtype != 'aws': assert tq.enqueued == n_inserts assert all(map(lambda x: type(x) == PrintTask, tq.list())) tq.purge() if qtype != 'aws': assert tq.enqueued == 0
def test_multi_threaded_insertion(): global QUEUE_NAME for qtype in QTYPES: tq = TaskQueue(n_threads=40, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL) n_inserts = 100 tq.purge() tq.wait() if qtype != 'aws': assert tq.enqueued == 0 for _ in range(n_inserts): task = PrintTask() tq.insert(task) tq.wait() if qtype == 'aws': list_len = 10 else: list_len = 100 lst = tq.list() assert len(lst) == list_len # task list api only lists 100 items at a time assert all(map(lambda x: type(x) == PrintTask, lst)) tq.purge() if qtype != 'aws': assert tq.enqueued == 0
def test_multi_threaded_insertion(sqs, protocol, green, threads): path = getpath(protocol) tq = TaskQueue(path, n_threads=threads, green=green) n_inserts = 40 tq.purge() tq.insert((PrintTask() for i in range(n_inserts))) tq.purge()
def test_parallel_insert_all(sqs, protocol): import pathos_issue path = getpath(protocol) tq = TaskQueue(path, green=True) tasks = pathos_issue.crt_tasks(5, 20) tq.insert(tasks, parallel=2) tq.purge()
def test_get(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) n_inserts = 5 tq.purge() tq.insert((PrintTask() for _ in range(n_inserts))) for i in range(n_inserts): t = tq.lease() tq.delete(t)
def test_single_threaded_insertion_fns(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) tq.purge() n_inserts = 5 tq.insert(( partial(printfn, "hello world " + str(i)) for i in range(n_inserts) )) assert all(map(lambda x: isinstance(x, FunctionTask), tq.list())) tq.purge()
def test_single_threaded_insertion(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) tq.purge() n_inserts = 5 tq.insert((PrintTask() for i in range(n_inserts))) assert all(map(lambda x: type(x) == PrintTask, tq.list())) tq.purge()
def test_get(): global QUEUE_NAME for qtype in QTYPES: tq = TaskQueue(n_threads=0, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL) n_inserts = 5 tq.purge() for _ in range(n_inserts): task = PrintTask() tq.insert(task) tq.wait() tq.purge()
def test_is_empty(): tq = TaskQueue(FILE_QURL) tq.purge() assert tq.is_empty() == True tq.insert(PrintTask("hello")) assert tq.is_empty() == False task = tq.lease() tq.delete(task) assert tq.is_empty() == True
def test_parallel_insert_all(sqs, protocol): import pathos_issue path = getpath(protocol) tq = TaskQueue(path, green=True) tq.purge() if protocol == 'fq': tq.rezero() tasks = pathos_issue.crt_tasks(5, 20) amt = tq.insert(tasks, parallel=2) assert amt == 15 if protocol == 'fq': assert tq.inserted == 15 tq.purge()
def test_enumerating_tasks(): tq = TaskQueue(FILE_QURL) tq.purge() for _ in range(10): tq.insert(PrintTask('hello')) tq.insert(PrintTask('world')) lst = list(tq.tasks()) assert len(lst) == 20 hello = 0 world = 0 for task in lst: hello += int(task.txt == "hello") world += int(task.txt == "world") assert hello == 10 assert world == 10
def test_polling(sqs): N = 100 tasks = [PrintTask(i) for i in range(N)] tq = TaskQueue(getpath('fq'), green=False) tq.purge() tq.insert(tasks) tq.poll(lease_seconds=1, verbose=False, tally=True, stop_fn=(lambda executed: executed >= 5)) tq.purge() tq.insert(tasks) tq.poll(lease_seconds=1, verbose=False, tally=True, stop_fn=(lambda elapsed_time: elapsed_time >= 1))
def test_queue_transfer(sqs): tqsqs = TaskQueue(getpath("sqs")) tqsqs.purge() tqfq = TaskQueue(getpath("fq")) tqfq.purge() assert tqsqs.enqueued == 0 tqfq.insert((PrintTask() for _ in range(10))) tqsqs.insert(tqfq) assert tqsqs.enqueued == 10 task = tqsqs.lease() assert isinstance(task, PrintTask) try: tqfq.insert(tqsqs) assert False except taskqueue.UnsupportedProtocolError: pass
def test_lease(sqs): path = getpath("sqs") tq = TaskQueue(path, n_threads=0) n_inserts = 20 tq.purge() tq.insert(( PrintTask(str(x)) for x in range(n_inserts) )) tasks = tq.lease(num_tasks=10, wait_sec=0) assert len(tasks) == 10 tq.delete(tasks) tasks = tq.lease(num_tasks=10, wait_sec=0) assert len(tasks) == 10 tq.delete(tasks) try: tasks = tq.lease(num_tasks=10, wait_sec=0) assert False except QueueEmptyError: pass
def purge(queuepath): """Delete all queued messages and zero out queue statistics.""" queuepath = normalize_path(queuepath) tq = TaskQueue(queuepath) tq.purge()