def crtq():
tq = TaskQueue(FILE_QURL)
tq.purge()
tq.rezero()
tq.insert((PrintTask(i) for i in range(N)))
return tq
def test_renew():
tq = TaskQueue(FILE_QURL)
tq.purge()
tq.insert(PrintTask('hello'))
ts = lambda fname: int(fname.split('--')[0])
ident = lambda fname: fname.split('--')[1]
filenames = os.listdir(tq.api.queue_path)
assert len(filenames) == 1
filename = filenames[0]
assert ts(filename) == 0
identity = ident(filename)
now = time.time()
tq.renew(filename, 1)
filenames = os.listdir(tq.api.queue_path)
assert len(filenames) == 1
filename = filenames[0]
assert ts(filename) >= int(time.time()) + 1
assert ident(filename) == identity
def test_single_threaded_insertion():
global QUEUE_NAME
for qtype in QTYPES:
tq = TaskQueue(n_threads=0, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL)
tq.purge()
if qtype != 'aws':
assert tq.enqueued == 0
n_inserts = 5
for _ in range(n_inserts):
task = PrintTask()
tq.insert(task)
tq.wait()
if qtype != 'aws':
assert tq.enqueued == n_inserts
assert all(map(lambda x: type(x) == PrintTask, tq.list()))
tq.purge()
if qtype != 'aws':
assert tq.enqueued == 0
def test_multi_threaded_insertion():
global QUEUE_NAME
for qtype in QTYPES:
tq = TaskQueue(n_threads=40, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL)
n_inserts = 100
tq.purge()
tq.wait()
if qtype != 'aws':
assert tq.enqueued == 0
for _ in range(n_inserts):
task = PrintTask()
tq.insert(task)
tq.wait()
if qtype == 'aws':
list_len = 10
else:
list_len = 100
lst = tq.list()
assert len(lst) == list_len # task list api only lists 100 items at a time
assert all(map(lambda x: type(x) == PrintTask, lst))
tq.purge()
if qtype != 'aws':
assert tq.enqueued == 0
def test_multi_threaded_insertion(sqs, protocol, green, threads):
path = getpath(protocol)
tq = TaskQueue(path, n_threads=threads, green=green)
n_inserts = 40
tq.purge()
tq.insert((PrintTask() for i in range(n_inserts)))
tq.purge()
def test_parallel_insert_all(sqs, protocol):
import pathos_issue
path = getpath(protocol)
tq = TaskQueue(path, green=True)
tasks = pathos_issue.crt_tasks(5, 20)
tq.insert(tasks, parallel=2)
tq.purge()
def test_get(sqs, protocol):
path = getpath(protocol)
tq = TaskQueue(path, n_threads=0)
n_inserts = 5
tq.purge()
tq.insert((PrintTask() for _ in range(n_inserts)))
for i in range(n_inserts):
t = tq.lease()
tq.delete(t)
def test_single_threaded_insertion_fns(sqs, protocol): path = getpath(protocol) tq = TaskQueue(path, n_threads=0) tq.purge() n_inserts = 5 tq.insert(( partial(printfn, "hello world " + str(i)) for i in range(n_inserts) )) assert all(map(lambda x: isinstance(x, FunctionTask), tq.list())) tq.purge()
def test_single_threaded_insertion(sqs, protocol):
path = getpath(protocol)
tq = TaskQueue(path, n_threads=0)
tq.purge()
n_inserts = 5
tq.insert((PrintTask() for i in range(n_inserts)))
assert all(map(lambda x: type(x) == PrintTask, tq.list()))
tq.purge()
def test_get():
global QUEUE_NAME
for qtype in QTYPES:
tq = TaskQueue(n_threads=0, queue_name=QUEUE_NAME, queue_server=qtype, qurl=QURL)
n_inserts = 5
tq.purge()
for _ in range(n_inserts):
task = PrintTask()
tq.insert(task)
tq.wait()
tq.purge()
def test_is_empty():
tq = TaskQueue(FILE_QURL)
tq.purge()
assert tq.is_empty() == True
tq.insert(PrintTask("hello"))
assert tq.is_empty() == False
task = tq.lease()
tq.delete(task)
assert tq.is_empty() == True
def test_parallel_insert_all(sqs, protocol):
import pathos_issue
path = getpath(protocol)
tq = TaskQueue(path, green=True)
tq.purge()
if protocol == 'fq':
tq.rezero()
tasks = pathos_issue.crt_tasks(5, 20)
amt = tq.insert(tasks, parallel=2)
assert amt == 15
if protocol == 'fq':
assert tq.inserted == 15
tq.purge()
def test_enumerating_tasks():
tq = TaskQueue(FILE_QURL)
tq.purge()
for _ in range(10):
tq.insert(PrintTask('hello'))
tq.insert(PrintTask('world'))
lst = list(tq.tasks())
assert len(lst) == 20
hello = 0
world = 0
for task in lst:
hello += int(task.txt == "hello")
world += int(task.txt == "world")
assert hello == 10
assert world == 10
def test_polling(sqs):
N = 100
tasks = [PrintTask(i) for i in range(N)]
tq = TaskQueue(getpath('fq'), green=False)
tq.purge()
tq.insert(tasks)
tq.poll(lease_seconds=1,
verbose=False,
tally=True,
stop_fn=(lambda executed: executed >= 5))
tq.purge()
tq.insert(tasks)
tq.poll(lease_seconds=1,
verbose=False,
tally=True,
stop_fn=(lambda elapsed_time: elapsed_time >= 1))
def test_queue_transfer(sqs):
tqsqs = TaskQueue(getpath("sqs"))
tqsqs.purge()
tqfq = TaskQueue(getpath("fq"))
tqfq.purge()
assert tqsqs.enqueued == 0
tqfq.insert((PrintTask() for _ in range(10)))
tqsqs.insert(tqfq)
assert tqsqs.enqueued == 10
task = tqsqs.lease()
assert isinstance(task, PrintTask)
try:
tqfq.insert(tqsqs)
assert False
except taskqueue.UnsupportedProtocolError:
pass
def test_lease(sqs):
path = getpath("sqs")
tq = TaskQueue(path, n_threads=0)
n_inserts = 20
tq.purge()
tq.insert(( PrintTask(str(x)) for x in range(n_inserts) ))
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert len(tasks) == 10
tq.delete(tasks)
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert len(tasks) == 10
tq.delete(tasks)
try:
tasks = tq.lease(num_tasks=10, wait_sec=0)
assert False
except QueueEmptyError:
pass
def purge(queuepath): """Delete all queued messages and zero out queue statistics.""" queuepath = normalize_path(queuepath) tq = TaskQueue(queuepath) tq.purge()
