class StarClusterTest(unittest.TestCase): __cfg = None @property def config(self): """ Returns (valid) default test config """ if not self.__cfg: tmp_file = tempfile.NamedTemporaryFile() tmp_file.write(config_test_template % default_config) tmp_file.flush() self.__cfg = StarClusterConfig(tmp_file.name, cache=True); self.__cfg.load() return self.__cfg def get_config(self, contents, cache=False): tmp_file = tempfile.NamedTemporaryFile() tmp_file.write(contents) tmp_file.flush() cfg = StarClusterConfig(tmp_file.name, cache=True); cfg.load() return cfg def get_custom_config(self, **kwargs): """ Returns default test config modified by kwargs """ kwords = {}; kwords.update(default_config) kwords.update(kwargs) cfg = self.get_config(config_test_template % kwords) return cfg
def test_config_dne(self): tmp_file = tempfile.NamedTemporaryFile() non_existent_file = tmp_file.name tmp_file.close() assert not os.path.exists(non_existent_file) try: cfg = StarClusterConfig(non_existent_file, cache=True); cfg.load() except exception.ConfigNotFound,e: pass
def get_custom_config(self, **kwargs): """ Returns default test config modified by kwargs """ tmp_file = tempfile.NamedTemporaryFile() kwords = {}; kwords.update(default_config) kwords.update(kwargs) tmp_file.write(config_test_template % kwords); tmp_file.flush() cfg = StarClusterConfig(tmp_file.name, cache=True); cfg.load() return cfg
def test_invalid_config(self): """ Test that reading a non-INI formatted file raises an exception """ tmp_file = tempfile.NamedTemporaryFile() tmp_file.write("""<html>random garbage file with no section headings</html>""") tmp_file.flush() try: cfg = StarClusterConfig(tmp_file.name, cache=True); cfg.load() except exception.ConfigHasNoSections,e: pass
def test_empty_config(self): """ Test that reading an empty config generates no errors and that aws credentials can be read from the environment. """ aws_key = 'testkey' aws_secret_key = 'testsecret' os.environ['AWS_ACCESS_KEY_ID'] = aws_key os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_key tmp_file = tempfile.NamedTemporaryFile() cfg = StarClusterConfig(tmp_file.name, cache=True); cfg.load() assert cfg.aws['aws_access_key_id'] == aws_key assert cfg.aws['aws_secret_access_key'] == aws_secret_key
def start_cluster(request): gpunum = request.POST["gpunum"] cluster_size = int(math.sqrt(int(gpunum))) varlist = json.dumps(["acp", "hgt", "alp"]) var3dlist = json.dumps(["delt", "dl", "sc", "dd", "dw", "dflx"]) var3dlistflow = json.dumps(["uh,vh", "zhyb", "omg", "rr", "dc"]) root_path = STORAGE_ROOTPATH # use this for server # root_path = "C:/QLiu/RemoteVizNow/NOW_geoviz/" # use this for localhost conf_template_path = root_path + "config_template" conf_new_path = root_path + "config" with open(conf_template_path, 'r') as f: conf_template = f.read() new_conf = conf_template.replace("[MY_CLUSTER_SIZE]", str(cluster_size)) with open(conf_new_path, 'wb') as f: f.write(new_conf) cfg = StarClusterConfig(conf_new_path) cfg.load() cluster = cfg.get_cluster_template("geovizcluster") try: cluster.start() except Exception as e: print e while not cluster.is_cluster_up(): sleep(10) for node in cluster.nodes: cmd = "sudo /usr/bin/X :1 && screen" node.ssh.execute(cmd, source_profile=False, detach=True) return { "varlist": varlist, "var3dlist": var3dlist, "var3dlistflow": var3dlistflow, "startcluster": "yes", "region": "1", "instancetype": "1", "gpunum": gpunum, }
def terminate_cluster(request): gpunum = request.POST["gpunum"] varlist = json.dumps(["acp", "hgt", "alp"]) var3dlist = json.dumps(["delt", "dl", "sc", "dd", "dw", "dflx"]) var3dlistflow = json.dumps(["uh,vh", "zhyb", "omg", "rr", "dc"]) root_path = STORAGE_ROOTPATH # use this for server # root_path = "C:/QLiu/RemoteVizNow/NOW_geoviz/" # use this for localhost conf_new_path = root_path + "config" cfg = StarClusterConfig(conf_new_path) cfg.load() cluster = cfg.get_cluster_template("geovizcluster") try: cluster.terminate_cluster(force=True) except: cluster.terminate_cluster(force=True) return { "varlist": varlist, "var3dlist": var3dlist, "var3dlistflow": var3dlistflow, "terminatecluster": "yes", }
return True def _validate_keypair(self): key_location = self.key_location if not os.path.exists(key_location): raise exception.ClusterValidationError("key_location=%s does not exist." % key_location) elif not os.path.isfile(key_location): raise exception.ClusterValidationError("key_location=%s is not a file." % key_location) keyname = self.keyname conn = self.ec2 keypair = self.ec2.get_keypair_or_none(keyname) if not keypair: raise exception.ClusterValidationError("Account does not contain a key with keyname = %s. " % keyname) if self.zone: z = self.ec2.get_zone(self.zone) if keypair.region != z.region: raise exception.ClusterValidationError( "Keypair %s not in availability zone region %s" % (keyname, z.region) ) return True if __name__ == "__main__": from starcluster.config import StarClusterConfig cfg = StarClusterConfig() cfg.load() sc = cfg.get_cluster_template("smallcluster", "mynewcluster") if sc.is_valid(): sc.start(create=True)
#!/usr/bin/env python import os import sys from starcluster.config import StarClusterConfig print 'Simple wrapper script for s3fs (http://s3fs.googlecode.com/)' cfg = StarClusterConfig(); cfg.load() ec2 = cfg.get_easy_ec2() buckets = ec2.s3.get_buckets() counter = 0 for bucket in buckets: print "[%d] %s" % (counter,bucket.name) counter += 1 try: inp = int(raw_input('>>> Enter the bucket to mnt: ')) selection = buckets[inp].name print 'you selected: %s' % selection mountpt = raw_input('>>> please enter the mnt point: ') print 'mounting %s at: %s' % (selection,mountpt) except KeyboardInterrupt,e: print print 'Exiting...' sys.exit(1) try: os.system('s3fs %s -o accessKeyId=%s -o secretAccessKey=%s %s' % (selection, cfg.aws.get('aws_access_key_id'), cfg.aws.get('aws_secret_access_key'),mountpt))
def get_config(self, contents, cache=False): tmp_file = tempfile.NamedTemporaryFile() tmp_file.write(contents) tmp_file.flush() cfg = StarClusterConfig(tmp_file.name, cache=True); cfg.load() return cfg
def writeconfig_ray(request): if request.method == 'POST': config_txt = request.POST["configtxt"] root_path = STORAGE_ROOTPATH wwconfig_path = root_path + "wwconfig.txt" with open(wwconfig_path, 'wb') as f: f.write(config_txt) conf_new_path = root_path + "config" cfg = StarClusterConfig(conf_new_path) cfg.load() cluster = cfg.get_cluster_template("geovizcluster") tasknum = int(request.POST["tasknum"]) gpunum = math.sqrt(tasknum) datafile = "data/" + request.POST["datafile"] varname = "dd" if request.POST["varname3d"] and request.POST["varname3d"] != "": varname = request.POST["varname3d"] intrange = 80 curtime = 8 if request.POST["curtime"] and request.POST["curtime"] != "": curtime = int(request.POST["curtime"]) w, h = int(request.POST["wwwidth"]), int(request.POST["wwheight"]) wdiv, hdiv = w / gpunum, h / gpunum #node_alias_list = [] img_list = [] for index, node in enumerate(cluster.nodes): #node_alias_list.append(node.alias) # send wwconfig file #local_config_path = "/home/bitnami/apps/django/django_projects/geoviz/geoviz/geovizapp/static/data/wwconfig.txt" local_config_path = wwconfig_path remote_config_path = "/home/ubuntu/CollabViz/wwconfig.txt" node.ssh.switch_user("ubuntu") node.ssh.put(local_config_path, remote_config_path) imgoutpath = "image/sub/" remote_root_path = "/home/ubuntu/CollabViz/" remote_img_path = remote_root_path + imgoutpath tmp_imgs = [] eachround = int(math.ceil(tasknum * 1.0 / gpunum)) for i in range(index * eachround, (index + 1) * eachround): if i < tasknum: curgpu = i wrange, hrange = int(i / gpunum), int(i % gpunum) w1, h1 = int(wrange * wdiv), int(hrange * hdiv) w2, h2 = int((wrange + 1) * wdiv), int((hrange + 1) * hdiv) tmp_imgs.append("%d_%d_%d_%d.png" % (w1, h1, w2, h2)) img_list.append(tmp_imgs) cmd = "sudo /home/ubuntu/anaconda/bin/python %s%s %d %d %d %s %s %d %d" % ( remote_root_path, "NetCDFCUDARayCasting.py", tasknum, gpunum, index, datafile, varname, intrange, curtime) node.ssh.execute(cmd, source_profile=False, detach=True) local_img_path = root_path + "img/sub/" for index, node in enumerate(cluster.nodes): for img in img_list[index]: img_get = False img_get_path = remote_img_path + img while not img_get: if node.ssh.isfile(img_get_path): node.ssh.get(img_get_path, local_img_path + img) img_get = True cmd = "sudo rm %s" % img_get_path node.ssh.execute(cmd, source_profile=False, detach=True) ncvrimage = NCVRImage(filepath=local_img_path) final_ray_img_file = "ray_%s.png" % datetime.datetime.now().strftime( '%Y%m%d%H%H%M%S') final_ray_img_file_path = root_path + "img/" + final_ray_img_file ncvrimage.MergeImageFromFolder(local_img_path, final_ray_img_file_path, w, h) response_data = {"newimgfile": final_ray_img_file} return HttpResponse(json.dumps(response_data), content_type="application/json")
def writeconfig_flow(request): if request.method == 'POST': config_txt = request.POST["configtxt"] root_path = STORAGE_ROOTPATH wwconfig_path = root_path + "wwconfig.txt" with open(wwconfig_path, 'wb') as f: f.write(config_txt) conf_new_path = root_path + "config" cfg = StarClusterConfig(conf_new_path) cfg.load() cluster = cfg.get_cluster_template("geovizcluster") tasknum = int(request.POST["tasknum"]) gpunum = math.sqrt(tasknum) datafile = "data/" + request.POST["datafile"] varname1, varname2 = "uh", "vh" if request.POST["varnameflow"] and request.POST["varnameflow"] != "": varnameflow = request.POST["varnameflow"].split(",") varname1, varname2 = varnameflow[0], varnameflow[1] intrange = 80 tseq = 8 if request.POST["tseq"] and request.POST["tseq"] != "": curtime = int(request.POST["tseq"]) colorselect = "ocean" if request.POST["colorselect"] and request.POST["colorselect"] != "": colorselect = request.POST["colorselect"] totalclass = 20 if request.POST["totalclass"] and request.POST["totalclass"] != "": totalclass = int(request.POST["totalclass"]) w, h = int(request.POST["wwwidth"]), int(request.POST["wwheight"]) wdiv, hdiv = w / gpunum, h / gpunum #node_alias_list = [] img_list = [] for index, node in enumerate(cluster.nodes): #node_alias_list.append(node.alias) # send wwconfig file #local_config_path = "/home/bitnami/apps/django/django_projects/geoviz/geoviz/geovizapp/static/data/wwconfig.txt" local_config_path = wwconfig_path remote_config_path = "/home/ubuntu/CollabViz/wwconfig.txt" node.ssh.switch_user("ubuntu") node.ssh.put(local_config_path, remote_config_path) imgoutpath = "image/" remote_root_path = "/home/ubuntu/CollabViz/" remote_img_path = remote_root_path + imgoutpath tmp_imgs = [] for i in range(index * tasknum / gpunum, (index + 1) * tasknum / gpunum): tmp_imgs.append("flowgpu%d.png" % i) tmp_imgs.append("legend.png") img_list.append(tmp_imgs) cmd = "sudo DISPLAY=:1 /home/ubuntu/anaconda/bin/python %s%s %d %d %d %s %s %s %d %s %d" % ( remote_root_path, "NetCDFCUDAFlow.py", tasknum, gpunum, index, datafile, varname1, varname2, tseq, colorselect, totalclass) node.ssh.execute(cmd, source_profile=False, detach=True) local_img_path = root_path + "img/sub/" for index, node in enumerate(cluster.nodes): for img in img_list[index]: img_get = False img_get_path = remote_img_path + img while not img_get: if node.ssh.isfile(img_get_path): node.ssh.get(img_get_path, local_img_path + img) img_get = True cmd = "sudo rm %s" % img_get_path node.ssh.execute(cmd, source_profile=False, detach=True) ncvrimage = NCVRImage(filepath=local_img_path) final_flow_img_file = "flow_%s.png" % datetime.datetime.now().strftime( '%Y%m%d%H%H%M%S') final_flow_img_file_path = root_path + "img/" + final_flow_img_file ncvrimage.MergeImageFromFolder(local_img_path, final_flow_img_file_path, w, h) response_data = {"newimgfile": final_flow_img_file} return HttpResponse(json.dumps(response_data), content_type="application/json")