def dispatch(cluster_nodes, imageid, imagename, imagesize, degree):
clusternum = len(cluster_nodes)
if clusternum == 0:
return
if clusternum == 1:
cluster = cluster_nodes[0]
nodenum = len(cluster)
if (nodenum <= 1):
return
ts = libThreading.threads()
thr = []
for client in cluster[1:degree]:
thr.append((wgetimage, client, cluster[0], imageid, imagename,
imagepath, imagesize))
ts.set_func_list(thr)
ts.start()
cs = libThreading.threads()
thr = []
clients = cluster[degree:]
lists = get_split_list(len(clients), degree, clients)
for i, list in enumerate(lists):
ll = []
list.insert(0, cluster[i])
ll.append(list)
print '<x>', ll
thr.append((dispatch, ll, imageid, imagename, imagesize, degree))
cs.set_func_list(thr)
cs.start()
else:
list = get_split_list(clusternum, degree, cluster_nodes)
# cluster_nodes is a list of clusters. cluster_nodes[0][0] is a node
# list is a list of list of clusters. list[0][0][0] is a node
ts = libThreading.threads()
thr = []
for client in list[1:]:
thr.append((wgetimage, client[0][0], list[0][0][0], imageid,
imagename, imagepath, imagesize))
ts.set_func_list(thr)
ts.start()
cs = libThreading.threads()
thr = []
for clusters in list:
thr.append(
(dispatch, clusters, imageid, imagename, imagesize, degree))
cs.set_func_list(thr)
cs.start()
def dispatch(cluster_nodes, imageid, imagename, imagesize, degree):
clusternum = len(cluster_nodes)
if clusternum == 0:
return
if clusternum == 1:
cluster = cluster_nodes[0]
nodenum = len(cluster)
if (nodenum <= 1):
return
ts = libThreading.threads()
thr = []
for client in cluster[1 : degree]:
thr.append((wgetimage, client, cluster[0], imageid, imagename, imagepath, imagesize))
ts.set_func_list(thr)
ts.start()
cs = libThreading.threads()
thr = []
clients = cluster[degree : ]
lists = get_split_list(len(clients), degree, clients)
for i, list in enumerate(lists):
ll = []
list.insert(0, cluster[i])
ll.append(list)
print '<x>', ll
thr.append((dispatch, ll, imageid, imagename, imagesize, degree))
cs.set_func_list(thr)
cs.start()
else:
list = get_split_list(clusternum, degree, cluster_nodes)
# cluster_nodes is a list of clusters. cluster_nodes[0][0] is a node
# list is a list of list of clusters. list[0][0][0] is a node
ts = libThreading.threads()
thr = []
for client in list[1:]:
thr.append((wgetimage, client[0][0], list[0][0][0], imageid, imagename, imagepath, imagesize))
ts.set_func_list(thr)
ts.start()
cs = libThreading.threads()
thr = []
for clusters in list:
thr.append((dispatch, clusters, imageid, imagename, imagesize, degree))
cs.set_func_list(thr)
cs.start()
def create_instance(nodes, instance_name):
ts = libThreading.threads()
thr = []
for i, n in enumerate(nodes):
thr.append((create_instance_threading, n, instance_name[i]))
ts.set_func_list(thr)
ts.start()
def local_copy(nodes, headnode, num, imageid, imagename, instance_names):
ts = libThreading.threads()
thr = []
for i, pid in enumerate(nodes):
thr.append((local_copy_threading, pid, headnode, num[i], instance_names[i], imageid, imagename))
ts.set_func_list(thr)
ts.start()
def kill_httpd(nodes):
ts = libThreading.threads()
thr = []
for i, pid in enumerate(nodes):
thr.append((kill_httpd_threading, pid))
ts.set_func_list(thr)
ts.start()
def kill_httpd(nodes):
ts = libThreading.threads()
thr = []
for i, pid in enumerate(nodes):
thr.append((kill_httpd_threading, pid))
ts.set_func_list(thr)
ts.start()
def dispatch_image(nodes, cluster_list, image_nodes, headnode, imageid, imagename, compresscount, is_samelocation):
cluster_nodes = get_clusters(nodes, cluster_list, image_nodes)
if len(cluster_nodes) == 0:
return
cluster_nodes = handle_headnode(cluster_nodes, headnode, is_samelocation)
#print cluster_nodes
index = 0
global timestamp
timestamp = time.time()
sortedfiles = os.popen('ls -lS %s/%s | awk \'{print $9}\'' % (imagepath, imageid)).read().split('\n')[1:-1]
##check files names?
while index < compresscount:
ts = time.time()
sbt = libThreading.threads()
thr = []
global semaphores
semaphores = []
treedegree = snowballtrees
if compresscount - index < treedegree:
treedegree = compresscount - index
for i in range(treedegree):
nodes = sbtree_design(cluster_nodes, i, treedegree)
#print nodes
sem = gen_semaphore(nodes)
semaphores.append(sem)
thr.append((dispatch, nodes, imageid, '%s' % (sortedfiles[index]), i))
index += 1
sbt.set_func_list(thr)
sbt.start()
def local_decompress(nodes, headnode, num, imageid, imagename, instance_names):
ts = libThreading.threads()
thr = []
for i, pid in enumerate(nodes):
thr.append((local_decompress_threading, pid, headnode, num[i], instance_names[i], imageid, imagename))
ts.set_func_list(thr)
ts.start()
def create_instance(nodes, instance_name):
ts = libThreading.threads()
thr = []
for i, n in enumerate(nodes):
thr.append((create_instance_threading, n, instance_name[i]))
ts.set_func_list(thr)
ts.start()
def dispatch_nv(nodes, headnode, imageid, concurrent):
dispatch_nv_thread(headnode, imageid, concurrent)
ts = libThreading.threads()
thr = []
for n in nodes:
if n != headnode:
thr.append((dispatch_nv_thread, n, imageid, concurrent))
ts.set_func_list(thr)
ts.start()
def dispatch_nv(nodes, headnode, imageid, concurrent):
dispatch_nv_thread(headnode, imageid, concurrent)
ts = libThreading.threads()
thr = []
for n in nodes:
if n != headnode:
thr.append((dispatch_nv_thread, n, imageid, concurrent))
ts.set_func_list(thr)
ts.start()
def dispatch(cluster_nodes, imageid, imagename, semindex):
clusternum = len(cluster_nodes)
if clusternum == 0:
return
if clusternum == 1:
cluster = cluster_nodes[0]
nodenum = len(cluster)
if (nodenum <= 1):
return
ts = libThreading.threads()
thr = []
clients = cluster[degree : ]
lists = get_split_list(len(clients), clients)
#print lists
#print cluster
for i, client in enumerate(cluster[1 : degree]):
if i + 1 < len(lists):
next = [[cluster[i + 1]] + lists[i + 1]]
else:
next = []
semaphore = semaphores[semindex][cluster[0]]
thr.append((wgetimage, client, cluster[0], imageid, imagename, imagepath, next, semaphore, semindex))
if len(lists) > 0:
next = [[cluster[0]] + lists[0]]
thr.append((waitdispatch, next, imageid, imagename, semindex))
ts.set_func_list(thr)
ts.start()
else:
list = get_split_list(clusternum, cluster_nodes)
# cluster_nodes is a list of clusters. cluster_nodes[0][0] is a node
# list is a list of list of clusters. list[0][0][0] is a node
ts = libThreading.threads()
thr = []
for client in list[1:]:
semaphore = semaphores[semindex][list[0][0][0]]
thr.append((wgetimage, client[0][0], list[0][0][0], imageid, imagename, imagepath, client, semaphore, semindex))
thr.append((waitdispatch, list[0], imageid, imagename, semindex))
ts.set_func_list(thr)
ts.start()
def check_nodes(nodes, imageid, imagename, checksum, imagesize):
ts = libThreading.threads()
thr = []
for n in nodes:
thr.append((check_nodes_threading, n, imageid, imagename, checksum, imagesize))
ts.set_func_list(thr)
ts.start()
ret = ts.get_return()
newnodes = []
for key in ret.keys():
if not ret[key]:
newnodes.append(nodes[key])
else:
print '%s already have this Image' % (nodes[key])
## Put some newnodes info
return newnodes
def check_nodes(nodes, imageid, imagename, checksum, imagesizes, compresscount):
ts = libThreading.threads()
thr = []
for n in nodes:
thr.append((check_nodes_threading, n, imageid, imagename, checksum, imagesizes, compresscount))
ts.set_func_list(thr)
ts.start()
ret = ts.get_return()
newnodes = []
for key in ret.keys():
if not ret[key]:
newnodes.append(nodes[key])
else:
print '%s already have this Image' % (nodes[key])
## Put some newnodes info
return newnodes