class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
movs = loadMovids()
movs_length = loadLengths()
start = 0
end = 396901
#Put longer task at start.
if end == 396901:
order = np.argsort(movs_length[start:])
else:
order = np.argsort(movs_length[start:end])
#For statistics
totWork = 0
completed = 0
print("Master loading work queue.")
for i in np.arange(start, end)[order]:
totWork += 1
movid = movs[i]
self.work_queue.add_work(data=('Do stuff', movid))
print("Master loading work queue done.")
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, movidf = slave_return_data
if done:
if message == "FAILED":
self.work_queue.add_work(data=('Do stuff', movidf))
else:
completed += 1
print("Progress: {0}/{1}, took {2}. {3}".format(
str(completed).rjust(6),
str(totWork).rjust(6), str(datetime.now() - time),
message))
# sleep some time
sleeptime.sleep(0.001)
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
movs = loadMovids()
movs_length = loadLengths()
#13b: 30598-30695
start = 30598
end = 30695
#Put longer task at start.
order = np.argsort(movs_length[start:end])
#For statistics
totWork = end - start
completed = 0
for i in np.arange(start,end)[order]:
movid = movs[i]
#It's modified that task is appended at front
self.work_queue.add_work(data=('Do stuff', movid))
#for i in range(tasks):
# self.work_queue.add_work(data=('Do stuff', i))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, movidf = slave_return_data
if done:
if message == "FAILED":
self.work_queue.add_work(data=('Do stuff', movidf))
else:
completed += 1
print("Progress: {0}/{1}, took {2}. {3}"
.format(str(completed).rjust(6),str(totWork).rjust(6),str(datetime.now()-time),message))
# sleep some time
sleeptime.sleep(0.001)
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
for i in np.arange(MPI.COMM_WORLD.Get_size() - 1):
self.work_queue.add_work(data=('Do stuff', i))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message = slave_return_data
if done:
print("Slave {0} is clear".format(message))
# sleep some time
sleeptime.sleep(0.3)
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
movs = loadMovids()
#13b: np.arange(30598,30695)
#1584
totWork = 1584 - 0
completed = 0
shadno = 0
shadyes = 0
shadwhat = 0
for i in np.arange(0, 1584):
movid = movs[i]
self.work_queue.add_work(data=('Do stuff', movid))
#for i in range(tasks):
# self.work_queue.add_work(data=('Do stuff', i))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, n, y, w = slave_return_data
shadno += n
shadyes += y
shadwhat += w
if done:
completed += 1
print("Progress: {0}/{1}, took {2}. {3}".format(
str(completed).rjust(6), totWork,
str(datetime.now() - time), message),
end="\r")
# sleep some time
sleeptime.sleep(0.1)
print("Progress: {0}/{1}, took {2}. {3}".format(
str(completed).rjust(6), totWork, str(datetime.now() - time),
message))
print("FEIF return {0} true".format(shadyes))
print("FEIF return {0} false".format(shadno))
print("total of {0} detection have no contour".format(shadwhat))
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
totWork = 0#end - start
completed = 0
for i in np.arange(folds): #Fold
for j in np.arange(gammas): #Gamma
for k in np.arange(cs): #C
for n in np.arange(pcs):
totWork += 1
self.work_queue.add_work(data=('Do stuff', i, j, k, n))
scores = np.zeros((folds,gammas,cs,pcs))
#for i in range(tasks):
# self.work_queue.add_work(data=('Do stuff', i))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, i, j, k, n, score = slave_return_data
if done:
completed += 1
scores[i,j,k,n] = score
print("Progress: {0}/{1}, took {2}. {3}"
.format(str(completed).rjust(6),str(totWork).rjust(6),str(datetime.now()-time),message))
# sleep some time
sleeptime.sleep(0.001)
np.save("/afs/inf.ed.ac.uk/group/ug4-projects/s1413557/training/svmscore/scores", scores)
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
movs = loadMovids()
movs_length = loadLengths()
#13b: 30598-30695
#0: 0-24941
#1: 24941-49433
#2: 49433-74100
#3: 74100-99012
#4: 99012-123658
#5: 123658-148701
#6: 148701-173761
#7: 173761-198372
#8: 198372-223132
#9: 223132-247825
#a: 247825-272495
#b: 272495-297592
#c: 297592-322424
#d: 322424-347142
#e: 347142-372011
#f: 372011-396901
start = 0
end = 1584
#Put longer task at start.
if end == 396901:
order = np.argsort(np.append(movs_length[372011:396901-1],(movs_length[-1])))
else:
order = np.argsort(movs_length[start:end])
#For statistics
totWork = 0#end - start
completed = 0
#for i in np.arange(start,end)[order]:
idlist = np.array([396853,396857,396859,396860,396866,396876,396880,396892,396895,396900,
396823,396829,396843,396852,396862,396870,396882,396885,396889,396894,
396856,396863,396864,396868,396869,396893,396896,396897,396898,396899,
396624,396639,396662,396675,396728,396737,396759,396777,396790,396804,
396750,396760,396767,396784,396792,396793,396872,396877,396878,396886,
396824,396836,396839,396854,396858,396861,396865,396871,396874,396884,
396720,396733,396778,396815,396820,396827,396830,396834,396879,396890,
396840,396841,396845,396849,396850,396851,396867,396883,396887,396891,
396785,396802,396811,396822,396838,396844,396873,396875,396881,396888])
order = np.argsort(movs_length[idlist])
#forty = [112,180,272,285,330,447,469,474,498,500,517,527,545,550,622]
#fortyone = [10, 33, 75, 82, 101, 114, 182, 183, 275, 279, 282, 291, 306, 325, 338, 380, 420, 424]
for i in idlist[order]:
if False and earlyRemoval(movs[i], movs_length[i]):
idee = movs[i][0]
savepath = "/afs/inf.ed.ac.uk/group/ug4-projects/s1413557/features/{0}/{1}/{2}"
savepath = savepath.format(idee[0],idee[0:2],idee)
if not os.path.isfile(savepath+".COMPLETE.npy"):
print("Master removes movid {0} with early removal".format(movs[i][0]))
np.save(savepath+".COMPLETE",np.array([]))
else:
totWork += 1
movid = movs[i]
#It's modified that task is appended at front
self.work_queue.add_work(data=('Do stuff', movid))
#for i in range(tasks):
# self.work_queue.add_work(data=('Do stuff', i))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, movidf = slave_return_data
if done:
if message == "FAILED":
self.work_queue.add_work(data=('Do stuff', movidf))
else:
completed += 1
print("Progress: {0}/{1}, took {2}. {3}"
.format(str(completed).rjust(6),str(totWork).rjust(6),str(datetime.now()-time),message))
# sleep some time
sleeptime.sleep(0.001)
class MyApp(object):
def __init__(self, slaves):
self.master = Master(slaves)
self.work_queue = WorkQueue(self.master)
def terminate_slaves(self):
self.master.terminate_slaves()
def run(self):
time = datetime.now()
movs = loadMovids()
movs_length = loadLengths()
#13b: 30598-30695
#0: 0-24941
#1: 24941-49433
#2: 49433-74100
#3: 74100-99012
#4: 99012-123658
#5: 123658-148701
#6: 148701-173761
#7: 173761-198372
#8: 198372-223132
#9: 223132-247825
#a: 247825-272495
#b: 272495-297592
#c: 297592-322424
#d: 322424-347142
#e: 347142-372011
#f: 372011-396901
start = 322424
end = 396901
#Put longer task at start.
if end == 396901:
order = np.argsort(movs_length[start:])
else:
order = np.argsort(movs_length[start:end])
#For statistics
totWork = 0#end - start
completed = 0
#idl = np.hstack((np.arange(30598,30633),np.arange(30634,30638)))
#idl = np.array([396853,396857,396859,396860,396866,396876,396880,396892,396895,396900,
# 396823,396829,396843,396852,396862,396870,396882,396885,396889,396894,
# 396856,396863,396864,396868,396869,396893,396896,396897,396898,396899,
# 396624,396639,396662,396675,396728,396737,396759,396777,396790,396804,
# 396750,396760,396767,396784,396792,396793,396872,396877,396878,396886,
# 396824,396836,396839,396854,396858,396861,396865,396871,396874,396884,
# 396720,396733,396778,396815,396820,396827,396830,396834,396879,396890,
# 396840,396841,396845,396849,396850,396851,396867,396883,396887,396891,
# 396785,396802,396811,396822,396838,396844,396873,396875,396881,396888])
#order = np.argsort(movs_length[idl])
#for i in idl[order]:
for i in np.arange(start,end)[order]:
#Alternative to early removal, so it wont thrash stuff.
if movs_length[i] > 50000:
idee = movs[i][0]
savepath = "/afs/inf.ed.ac.uk/group/project/F4KC/output/{0}/{1}/{2}"
savepath = savepath.format(idee[0],idee[0:2],idee)
if not os.path.isfile(savepath+".COMPLETE.npy"):
print("Master removes movid {0} with early removal".format(movs[i][0]))
yhat = np.zeros((movs_length[i],4,10))
yhat[:,:,1] = 1
np.save(savepath+".YHAT",yhat)
np.save(savepath+".COMPLETE",np.array([]))
totWork += 1
movid = movs[i]
self.work_queue.add_work(data=('Do stuff', movid))
while not self.work_queue.done():
self.work_queue.do_work()
#
# reclaim returned data from completed slaves
#
for slave_return_data in self.work_queue.get_completed_work():
done, message, movidf = slave_return_data
if done:
if message == "FAILED":
self.work_queue.add_work(data=('Do stuff', movidf))
else:
completed += 1
print("Progress: {0}/{1}, took {2}. {3}"
.format(str(completed).rjust(6),str(totWork).rjust(6),str(datetime.now()-time),message))
# sleep some time
sleeptime.sleep(0.001)