def start_cluster(request):
gpunum = request.POST["gpunum"]
cluster_size = int(math.sqrt(int(gpunum)))
varlist = json.dumps(["acp", "hgt", "alp"])
var3dlist = json.dumps(["delt", "dl", "sc", "dd", "dw", "dflx"])
var3dlistflow = json.dumps(["uh,vh", "zhyb", "omg", "rr", "dc"])
root_path = STORAGE_ROOTPATH # use this for server
# root_path = "C:/QLiu/RemoteVizNow/NOW_geoviz/" # use this for localhost
conf_template_path = root_path + "config_template"
conf_new_path = root_path + "config"
with open(conf_template_path, 'r') as f:
conf_template = f.read()
new_conf = conf_template.replace("[MY_CLUSTER_SIZE]", str(cluster_size))
with open(conf_new_path, 'wb') as f:
f.write(new_conf)
cfg = StarClusterConfig(conf_new_path)
cfg.load()
cluster = cfg.get_cluster_template("geovizcluster")
try:
cluster.start()
except Exception as e:
print e
while not cluster.is_cluster_up():
sleep(10)
for node in cluster.nodes:
cmd = "sudo /usr/bin/X :1 && screen"
node.ssh.execute(cmd, source_profile=False, detach=True)
return {
"varlist": varlist,
"var3dlist": var3dlist,
"var3dlistflow": var3dlistflow,
"startcluster": "yes",
"region": "1",
"instancetype": "1",
"gpunum": gpunum,
}
def terminate_cluster(request):
gpunum = request.POST["gpunum"]
varlist = json.dumps(["acp", "hgt", "alp"])
var3dlist = json.dumps(["delt", "dl", "sc", "dd", "dw", "dflx"])
var3dlistflow = json.dumps(["uh,vh", "zhyb", "omg", "rr", "dc"])
root_path = STORAGE_ROOTPATH # use this for server
# root_path = "C:/QLiu/RemoteVizNow/NOW_geoviz/" # use this for localhost
conf_new_path = root_path + "config"
cfg = StarClusterConfig(conf_new_path)
cfg.load()
cluster = cfg.get_cluster_template("geovizcluster")
try:
cluster.terminate_cluster(force=True)
except:
cluster.terminate_cluster(force=True)
return {
"varlist": varlist,
"var3dlist": var3dlist,
"var3dlistflow": var3dlistflow,
"terminatecluster": "yes",
}
return True
def _validate_keypair(self):
key_location = self.key_location
if not os.path.exists(key_location):
raise exception.ClusterValidationError("key_location=%s does not exist." % key_location)
elif not os.path.isfile(key_location):
raise exception.ClusterValidationError("key_location=%s is not a file." % key_location)
keyname = self.keyname
conn = self.ec2
keypair = self.ec2.get_keypair_or_none(keyname)
if not keypair:
raise exception.ClusterValidationError("Account does not contain a key with keyname = %s. " % keyname)
if self.zone:
z = self.ec2.get_zone(self.zone)
if keypair.region != z.region:
raise exception.ClusterValidationError(
"Keypair %s not in availability zone region %s" % (keyname, z.region)
)
return True
if __name__ == "__main__":
from starcluster.config import StarClusterConfig
cfg = StarClusterConfig()
cfg.load()
sc = cfg.get_cluster_template("smallcluster", "mynewcluster")
if sc.is_valid():
sc.start(create=True)
def writeconfig_ray(request):
if request.method == 'POST':
config_txt = request.POST["configtxt"]
root_path = STORAGE_ROOTPATH
wwconfig_path = root_path + "wwconfig.txt"
with open(wwconfig_path, 'wb') as f:
f.write(config_txt)
conf_new_path = root_path + "config"
cfg = StarClusterConfig(conf_new_path)
cfg.load()
cluster = cfg.get_cluster_template("geovizcluster")
tasknum = int(request.POST["tasknum"])
gpunum = math.sqrt(tasknum)
datafile = "data/" + request.POST["datafile"]
varname = "dd"
if request.POST["varname3d"] and request.POST["varname3d"] != "":
varname = request.POST["varname3d"]
intrange = 80
curtime = 8
if request.POST["curtime"] and request.POST["curtime"] != "":
curtime = int(request.POST["curtime"])
w, h = int(request.POST["wwwidth"]), int(request.POST["wwheight"])
wdiv, hdiv = w / gpunum, h / gpunum
#node_alias_list = []
img_list = []
for index, node in enumerate(cluster.nodes):
#node_alias_list.append(node.alias)
# send wwconfig file
#local_config_path = "/home/bitnami/apps/django/django_projects/geoviz/geoviz/geovizapp/static/data/wwconfig.txt"
local_config_path = wwconfig_path
remote_config_path = "/home/ubuntu/CollabViz/wwconfig.txt"
node.ssh.switch_user("ubuntu")
node.ssh.put(local_config_path, remote_config_path)
imgoutpath = "image/sub/"
remote_root_path = "/home/ubuntu/CollabViz/"
remote_img_path = remote_root_path + imgoutpath
tmp_imgs = []
eachround = int(math.ceil(tasknum * 1.0 / gpunum))
for i in range(index * eachround, (index + 1) * eachround):
if i < tasknum:
curgpu = i
wrange, hrange = int(i / gpunum), int(i % gpunum)
w1, h1 = int(wrange * wdiv), int(hrange * hdiv)
w2, h2 = int((wrange + 1) * wdiv), int((hrange + 1) * hdiv)
tmp_imgs.append("%d_%d_%d_%d.png" % (w1, h1, w2, h2))
img_list.append(tmp_imgs)
cmd = "sudo /home/ubuntu/anaconda/bin/python %s%s %d %d %d %s %s %d %d" % (
remote_root_path, "NetCDFCUDARayCasting.py", tasknum, gpunum,
index, datafile, varname, intrange, curtime)
node.ssh.execute(cmd, source_profile=False, detach=True)
local_img_path = root_path + "img/sub/"
for index, node in enumerate(cluster.nodes):
for img in img_list[index]:
img_get = False
img_get_path = remote_img_path + img
while not img_get:
if node.ssh.isfile(img_get_path):
node.ssh.get(img_get_path, local_img_path + img)
img_get = True
cmd = "sudo rm %s" % img_get_path
node.ssh.execute(cmd,
source_profile=False,
detach=True)
ncvrimage = NCVRImage(filepath=local_img_path)
final_ray_img_file = "ray_%s.png" % datetime.datetime.now().strftime(
'%Y%m%d%H%H%M%S')
final_ray_img_file_path = root_path + "img/" + final_ray_img_file
ncvrimage.MergeImageFromFolder(local_img_path, final_ray_img_file_path,
w, h)
response_data = {"newimgfile": final_ray_img_file}
return HttpResponse(json.dumps(response_data),
content_type="application/json")
def writeconfig_flow(request):
if request.method == 'POST':
config_txt = request.POST["configtxt"]
root_path = STORAGE_ROOTPATH
wwconfig_path = root_path + "wwconfig.txt"
with open(wwconfig_path, 'wb') as f:
f.write(config_txt)
conf_new_path = root_path + "config"
cfg = StarClusterConfig(conf_new_path)
cfg.load()
cluster = cfg.get_cluster_template("geovizcluster")
tasknum = int(request.POST["tasknum"])
gpunum = math.sqrt(tasknum)
datafile = "data/" + request.POST["datafile"]
varname1, varname2 = "uh", "vh"
if request.POST["varnameflow"] and request.POST["varnameflow"] != "":
varnameflow = request.POST["varnameflow"].split(",")
varname1, varname2 = varnameflow[0], varnameflow[1]
intrange = 80
tseq = 8
if request.POST["tseq"] and request.POST["tseq"] != "":
curtime = int(request.POST["tseq"])
colorselect = "ocean"
if request.POST["colorselect"] and request.POST["colorselect"] != "":
colorselect = request.POST["colorselect"]
totalclass = 20
if request.POST["totalclass"] and request.POST["totalclass"] != "":
totalclass = int(request.POST["totalclass"])
w, h = int(request.POST["wwwidth"]), int(request.POST["wwheight"])
wdiv, hdiv = w / gpunum, h / gpunum
#node_alias_list = []
img_list = []
for index, node in enumerate(cluster.nodes):
#node_alias_list.append(node.alias)
# send wwconfig file
#local_config_path = "/home/bitnami/apps/django/django_projects/geoviz/geoviz/geovizapp/static/data/wwconfig.txt"
local_config_path = wwconfig_path
remote_config_path = "/home/ubuntu/CollabViz/wwconfig.txt"
node.ssh.switch_user("ubuntu")
node.ssh.put(local_config_path, remote_config_path)
imgoutpath = "image/"
remote_root_path = "/home/ubuntu/CollabViz/"
remote_img_path = remote_root_path + imgoutpath
tmp_imgs = []
for i in range(index * tasknum / gpunum,
(index + 1) * tasknum / gpunum):
tmp_imgs.append("flowgpu%d.png" % i)
tmp_imgs.append("legend.png")
img_list.append(tmp_imgs)
cmd = "sudo DISPLAY=:1 /home/ubuntu/anaconda/bin/python %s%s %d %d %d %s %s %s %d %s %d" % (
remote_root_path, "NetCDFCUDAFlow.py", tasknum, gpunum, index,
datafile, varname1, varname2, tseq, colorselect, totalclass)
node.ssh.execute(cmd, source_profile=False, detach=True)
local_img_path = root_path + "img/sub/"
for index, node in enumerate(cluster.nodes):
for img in img_list[index]:
img_get = False
img_get_path = remote_img_path + img
while not img_get:
if node.ssh.isfile(img_get_path):
node.ssh.get(img_get_path, local_img_path + img)
img_get = True
cmd = "sudo rm %s" % img_get_path
node.ssh.execute(cmd,
source_profile=False,
detach=True)
ncvrimage = NCVRImage(filepath=local_img_path)
final_flow_img_file = "flow_%s.png" % datetime.datetime.now().strftime(
'%Y%m%d%H%H%M%S')
final_flow_img_file_path = root_path + "img/" + final_flow_img_file
ncvrimage.MergeImageFromFolder(local_img_path,
final_flow_img_file_path, w, h)
response_data = {"newimgfile": final_flow_img_file}
return HttpResponse(json.dumps(response_data),
content_type="application/json")