def main():
held_taskq = Queue.Queue()
taskslist=[]
count = 1
highq = sys.argv[0]
lowq = sys.argv[1]
totaltasks = sys.argv[2]
for i in range(int(totaltasks)):
arg1 = randint(0,100)
arg2 = randint(0,100)
priority = pick_random([(0,.2),(1,.8)])
if priority == 0:
count = count + 1
task = (add.s(arg1,arg2),1)
else:
task = (add.s(arg1,arg2),1)
taskslist.append(task)
#held_taskq.put(task)
#count = count + 1
#print "total high tasks ",count
FILE = open("tasks5.txt", 'w')
cPickle.dump(taskslist, FILE)
FILE.close()
FILE = open("tasks5.txt", 'r')
taskslist = cPickle.load(FILE)
FILE.close()
print "Task list"
print taskslist
for item in taskslist:
held_taskq.put(item)
cond = Condition()
t = my_monitor(held_taskq,highq,lowq,cond) #instance for the my_monitor class
t.daemon = True
t.start()
t.test()
def main():
loop = []
# count1 = 0
# count2 = 0
#code to generate 300 tasks of random types
# for x in xrange(300):
# opertn = randint(0,2)
# if opertn == 0:
# temp = add.s((randint(0,100)),(randint(0,100)))
# print temp
# count = count + 1
# count1 = count1 + 1
# loop.append(temp)
# elif opertn == 1:
# temp = add1.s((randint(0,100)),(randint(0,100)))
# print temp
# count = count + 1
# count2 = count2 + 1
# loop.append(temp)
# else:
# temp = add2.s((randint(0,100)),(randint(0,100)))
# print temp
# loop.append(temp)
# print " add ",count1," add1 ",count2
# code to generate 300 tasks of same type
count1 = 0
for x in xrange(300):
temp = add.s((randint(0,100)),(randint(0,100)))
print temp
count1 = count1+1
loop.append(temp)
FILE = open("tasks1.txt", 'w')
cPickle.dump(loop, FILE)
FILE.close()
FILE = open("tasks1.txt", 'r')
taskslist = cPickle.load(FILE)
FILE.close()
print "Task list"
print taskslist
schedulertype = sys.argv[0]
# print "total add tasks ",count1
if schedulertype == "FCFS":
schedulerObj = FCFSScheduler(["wnode1","wnode2","wnode3"])
else:
schedulerObj = RandomizedScheduler(["wnode1","wnode2","wnode3"])
schedulerObj.assignTasks("client1",taskslist)
def main():
loop = []
# count1 = 0
# count2 = 0
#code to generate 300 tasks of random types
# for x in xrange(300):
# opertn = randint(0,2)
# if opertn == 0:
# temp = add.s((randint(0,100)),(randint(0,100)))
# print temp
# count = count + 1
# count1 = count1 + 1
# loop.append(temp)
# elif opertn == 1:
# temp = add1.s((randint(0,100)),(randint(0,100)))
# print temp
# count = count + 1
# count2 = count2 + 1
# loop.append(temp)
# else:
# temp = add2.s((randint(0,100)),(randint(0,100)))
# print temp
# loop.append(temp)
# print " add ",count1," add1 ",count2
# code to generate 300 tasks of same type
count1 = 0
for x in xrange(300):
temp = add.s((randint(0, 100)), (randint(0, 100)))
print temp
count1 = count1 + 1
loop.append(temp)
FILE = open("tasks1.txt", 'w')
cPickle.dump(loop, FILE)
FILE.close()
FILE = open("tasks1.txt", 'r')
taskslist = cPickle.load(FILE)
FILE.close()
print "Task list"
print taskslist
schedulertype = sys.argv[0]
# print "total add tasks ",count1
if schedulertype == "FCFS":
schedulerObj = FCFSScheduler(["wnode1", "wnode2", "wnode3"])
else:
schedulerObj = RandomizedScheduler(["wnode1", "wnode2", "wnode3"])
schedulerObj.assignTasks("client1", taskslist)
def main():
held_taskq = Queue.Queue()
taskslist = []
count = 1
highq = sys.argv[0]
lowq = sys.argv[1]
totaltasks = sys.argv[2]
for i in range(int(totaltasks)):
arg1 = randint(0, 100)
arg2 = randint(0, 100)
priority = pick_random([(0, .2), (1, .8)])
if priority == 0:
count = count + 1
task = (add.s(arg1, arg2), 1)
else:
task = (add.s(arg1, arg2), 1)
taskslist.append(task)
#held_taskq.put(task)
#count = count + 1
#print "total high tasks ",count
FILE = open("tasks5.txt", 'w')
cPickle.dump(taskslist, FILE)
FILE.close()
FILE = open("tasks5.txt", 'r')
taskslist = cPickle.load(FILE)
FILE.close()
print "Task list"
print taskslist
for item in taskslist:
held_taskq.put(item)
cond = Condition()
t = my_monitor(held_taskq, highq, lowq,
cond) #instance for the my_monitor class
t.daemon = True
t.start()
t.test()
def test():
#res = add.s(2,2).delay()
#print res.id
#print res.state
#print res.get()
#print res.state
#task_list = [add.s(i,i) for i in xrange(10)]
#print task_list
#ch = chain(task_list)
ch = chain(add.s(2, 2), mul.s(3))
print ch
res = ch.delay()
print res
print res.id
print res.state
res.get(5)
print res.state
print res.result
print res.info
tasks : list
List of Celery tasks to choose from.
delay : int or float (optional)
Delay between execution of two successive Celery tasks.
queue : string
Name of the queue
"""
while True:
t = random.choice(tasks)
print "Executing task: {}".format(t)
t.apply_async(queue=queue)
time.sleep(delay)
# A list of short-running tasks
short_running_tasks = [add.s(2, 2), mul.s(10, 12), xsum.s(range(10))]
# A list of long-running tasks
long_running_tasks = [fib.s(35), fib.s(36), fib.s(37)]
short_t = threading.Thread(target=run_tasks, args=(short_running_tasks,),
kwargs={"delay" : 1, "queue" : "hipri"})
long_t = threading.Thread(target=run_tasks, args=(long_running_tasks,),
kwargs={"delay" : 5})
for t in [short_t, long_t]:
# set thread to a daemon thread so that it doesn't block the main program
# from exiting
t.daemon = True
t.start()
# Wait for an interrupt signal.
from proj.tasks import add, div, sub, avg
from proj.celery import app
from celery import group, chain, chord
g = group(add.s(i, i)
for i in range(10))().get() #for group we must to use signature
print(g)
'''partial
g = group(add.s(i, i)for i in range(10))
print(g(10).get())
'''
#FoG math
ch = chain(add.s(2, 4) | add.s(2))
print(ch().get())
ch = chain(add.s(4) | div.s(2))
print(ch(6).get())
ch = chord((add.s(i, i) for i in range(10)), avg.s())
print(ch().get())
ch = (group(add.s(i, i) for i in range(10)) | avg.s() | add.s(2))
print(ch().get())
''' #--------------------------以下是test---------------------------- from proj.tasks import add print(add(3,4)) res = add.delay(2, 2) print(res.get(timeout=1)) s1 = add.subtask((2, 2), countdown=10) res = s1.delay() res.get() # add options s3 = add.s(2, 2, debug=True) s3.delay(debug=False) # debug is now False. # Groups group(add.s(i, i) for i in xrange(10))().get() # Partial group g = group(add.s(i) for i in xrange(10)) g(10).get() # Chains (4 + 4) * 8 chain(add.s(4, 4) | mul.s(8))().get() # Chords # A chord is a group with a callback:
from celery import group, chain, chord, signature
from proj.tasks import add, mul, xsum, hello
print(group(add.s(i, i) for i in range(10))().get())
g = group(add.s(i) for i in range(200))
print(g(10).get())
# (4 + 4) * 9
print(chain(add.s(4, 4) | mul.s(9))().get())
# (? + 4) * 8
c = chain(add.s(4) | mul.s(8))
print(c(4).get())
print(chord((add.s(i, i) for i in range(10)), xsum.s())().get())
a = add.apply_async(args=(1, 1), kwargs={}, countdown=10, expires=120)
r = add.s(1, 1).apply_async(countdown=10)
print(r.state)
def on_raw_message(body):
print(body)
r = hello.apply_async(args=(3, 5))
print(r.get(on_message=on_raw_message, propagate=False))
r = signature('tasks.add', args=(2, 2), countdown=10)
print(r)
r = add.signature((2, 2), {}, countdown=10)
# print(add(2, 2))
print('@@@@@@@@@@@@@@@@@@@@@@@@')
print(result.failed())
print(result.id)
print(result.backend)
print(result.collect())
print(result.ready())
print(result.get(propagate=False))
# print(result.ready())
print(result.failed())
print('@@@@@@@@@@@@@@@@@@@@@@@@')
func = add.s(2, 2)
print(type(func))
print(func)
res = func.delay()
print(res.get())
print('@@@@@@@@@@@@@@@@@@@@@@@@')
res = group(add.s(i, i) for i in range(10))().get()
print(res)
g = group(add.s(i) for i in range(10))
print(g(10).get())
print('@@@@@@@@@@@@@@@@@@@@@@@@')
# (4 + 4) * 8
from proj.tasks import add # result=add.delay(4,4) result=add.apply_async((4,4)) print(result.ready()) print(result.get(timeout=1)) s1=add.s(4,4) result=s1.delay() print(result.get())
from celery import group, chain, chord
from proj.tasks import add, mul, xsum
if __name__ == '__main__':
# (i + i) for i in range 100
for i in range(100):
result = add.delay(i, i)
print(result.get(propagate=False))
# signatures
# (8 + 2)
s1 = add.s(2)
res = s1.delay(8)
print(res.get())
# groups
# list of (10 + i) for i in range 10
g = group(add.s(i) for i in range(10))
print(g(10).get())
# chains
# (? + 4) * 8
c = chain(add.s(4) | mul.s(8))
print(c(4).get())
# chord
# sum of all i + i in range 10
m = chord((add.s(i, i) for i in range(10)), xsum.s())().get()
print(m)
from celery import chain from proj.tasks import add res = chain(add.s(2, 2), add.s(4), add.s(8))() res.get()
(add.delay(rand1, rand2).get(), mul.delay(rand1, rand2).get())
)
print(ans_pool)
##############
# Multiple CPU + Group/Chord
##############
# Prepare data and assign tasks
prepare_add = []
prepare_mul = []
for i in range(0, 10):
rand1 = random.randint(0, 99)
rand2 = random.randint(0, 99)
prepare_add.append(add.s(rand1, rand2))
prepare_mul.append(mul.s(rand1, rand2))
# Group tasks
group_add = group(prepare_add)
group_mul = group(prepare_mul)
# Apply async (waste few time)
result_add = group_add.apply_async()
result_mul = group_mul.apply_async()
result_sum = (group(prepare_add) | xsum.s())()
# Get result (waste lots time)
print([item.get() for item in result_add])
print([item.get() for item in result_mul])
print(result_sum.get())
from celery import group, chord, chain from celery.task.control import discard_all from proj.tasks import random_int, add, mul, xsum discard_all() # Chain res = chain(add.s(4, 4), mul.s(8), mul.s(10))() print res.get() # Chord Example callback = xsum.s() header = [add.s(i,i) for i in range(0,10)] result = chord(header)(callback) result = result.get() print result # Group Example job = group(random_int.s(i) for i in range(0,10)) result = job.apply_async() while not result.ready(): pass print result.ready()