def run(): ClusterManager.init() cluster_id = ClusterManager.createCluster(CLUSTER_NAME, write_DB=False) cluster_id = cluster_id.data.get("cluster_id") ClusterManager.addNode(cluster_id, NODE_NAME, write_DB=False) wrong_cluster_id = "wrong id" try: result = ClusterManager.deleteNode(wrong_cluster_id, NODE_NAME[0], write_DB=False) if result.code == "failed": return True else: return False except: return False finally: ClusterManager.deleteNode(cluster_id, "compute1", write_DB=False)
def run(): ClusterManager.init() cluster_id = ClusterManager.createCluster(CLUSTER_NAME, write_DB=False) cluster_id = cluster_id.data.get("cluster_id") ClusterManager.addNode(cluster_id, NODE_NAME, write_DB=False) try: result = ClusterManager.listNode(cluster_id) node_list = result.data.get("node_list") if len(node_list) == 1: return True else: return False except: return False finally: ClusterManager.deleteNode(cluster_id, "compute1", write_DB=False)
def run(): ClusterManager.init() instance_id = Preprocess.create_with_provider_instance() cluster_id = ClusterManager.createCluster(CLUSTER_NAME, write_DB=False) cluster_id = cluster_id.data.get("cluster_id") ClusterManager.addNode(cluster_id, NODE_NAME, write_DB=False) try: result = ClusterManager.addInstance(cluster_id, instance_id, write_DB=False, send_flag=False) if result.code == "succeed": return True else: return False except: return False finally: ClusterManager.deleteNode(cluster_id, "compute1", write_DB=False) Postprocess.deleteInstance()
def deleteNode(self, cluster_id, node_name): """ The function for delete a computing node from HA cluster. Put the cluster uuid and node name to this function, it will delete node from HA cluster. Args: cluster_Id (str): cluster uuid. node_name (str): node name. Return: (map) delete node result. {"code" : "0","message": message} -> success. {"code" : "1","message": message} -> fail. """ try: result = ClusterManager.deleteNode(cluster_id, node_name) return result except: logging.error("HASS--delete node fail")
def addNode(self, clusterId, nodeList): """ The function for add a computing node to HA cluster. Put the cluster uuid and nodeList to this function, it will add node to HA cluster. Args: cluster_Id (str): cluster uuid. nodeList (str): node name. Return: (map) add node result. {"code" : "0","message": message} -> success. {"code" : "1","message": message} -> fail. """ try: result = ClusterManager.addNode(clusterId, nodeList) return result except: logging.error("HASS--add node fail")
def deleteInstance(self, clusterId, instanceId): """ The function for delete a instance from HA cluster. Put the cluster uuid and instance id to this function, it will delete instance from HA cluster. Args: clusterId (str): cluster uuid. instanceId (str): instance id. Return: (map) delete instance result. {"code" : "0","message": message} -> success. {"code" : "1","message": message} -> fail. """ try: result = ClusterManager.deleteInstance(clusterId, instanceId) logging.info("HASS--delete instance success") return result except: logging.error("HASS--delete instance fail")
def recoverSensorCritical(self, cluster_id, fail_node_name): """ :param cluster_id: :param fail_node_name: :return: """ cluster = ClusterManager.getCluster(cluster_id) if not cluster: logging.error("RecoverManager : cluster not found") return fail_node = cluster.getNodeByName(fail_node_name) print("fail node is %s" % fail_node.name) print("start recovery vm") self.recoverVMByEvacuate(cluster, fail_node) # self.recoverVMByLiveMigrate(cluster, fail_node) print("end evacuate vm") return self.recoverNodeByShutoff(fail_node)
def run(): ClusterManager.init() cluster_id = ClusterManager.createCluster(CLUSTER_NAME, write_DB=False) cluster_id = cluster_id.data.get("cluster_id") try: result = ClusterManager.addNode(cluster_id, NODE_NAME, write_DB=False) if result.code == "succeed": return True else: return False except: return False finally: ClusterManager.deleteNode(cluster_id, NODE_NAME[0], write_DB=False)
def recoverSensorCriticalByConfig(self, cluster_id, fail_node_name): """ :param cluster_id: :param fail_node_name: :return: """ print("start recover sensor:", time.time()) cluster = ClusterManager.getCluster(cluster_id) if not cluster: logging.error("RecoverManager : cluster not found") return fail_node = cluster.getNodeByName(fail_node_name) print("start recover vm:", time.time()) self.recoverVMByLiveMigrate(cluster, fail_node) # self.recoverVMByEvacuate(cluster, fail_node) # test print("end recover vm and start reboot host", time.time()) # return self.recoverNodeByReboot(fail_node) temp = self.recoverNodeByReboot(fail_node) print("finish recover host:", time.time()) return temp
node = ring[data['key']] print(f"Sending data:{data}") node.send_json(data) node.recv_json() time.sleep(1) #Send remove signal to Consul cm.deregisterNode(nodeName=nodeName) print("Done") if __name__ == "__main__": servers = [] cm = ClusterManager() nodes = cm.listNodes() for nodeName, node in nodes.items(): port = node['Port'] servers.append(f'tcp://127.0.0.1:{port}') print("Servers:", servers) # Create Consistent Hash ring ring = create_consistent_hashing_ring(servers) # Phase 1 Code # generate_data_round_robin(servers) # generate_data_consistent_hashing(servers)
def __init__(self): self.tc=0; self.cluster_manager=ClusterManager() self.grid_list=GridList() self.gap = Helper().gap()
class D_Stream: #=================变量声明======================= #时间尺度里的当前时间点,初始值是0,之后以整数1为单位推进 tc=0 #grid_list,用于存储grid的character_vector的dic grid_list=None #clusters 类簇的记录表,kv结构,v集合,集合里保存key #clusters={} #ClusterManager cluster_manager=None gap=0 #========以下是函数================= #TODO:初始化变量,结构等,做好准备工作 def __init__(self): self.tc=0; self.cluster_manager=ClusterManager() self.grid_list=GridList() self.gap = Helper().gap() #注释:现在直接往cluster里加grid就可以 # def __addToCluster(self,grid,cluster): # #到cluster里 # #更新操作clusters表 def __adjust_sparse(self,grid_object): #TODO: logging.warn("the __adjust_sparse was been removed") return #把这个grid从cluster中移除 cluster_object=self.cluster_manager.getCluster(grid_object.clusterKey()) cluster_object.delGrid(grid_object) if cluster_object.size()==0: self.cluster_manager.delCluster(cluster_object.key()) if not cluster_object.isClusterSingle(): self.cluster_manager.splitCluster(cluster_object.key()) #================================================= def __adjust_dense_neighbor_dense(self,grid_object,grid_h_object): #如果g还没有cluster,调用__addToClusters if -1==grid_object.clusterKey(): logging.debug("grid "+grid_object.key()+" has not been cluster") grid_h_cluster_object=self.cluster_manager.getCluster(grid_h_object.clusterKey()) grid_h_cluster_object.addGrid(grid_object) #如果g已经有cluster且他的cluseter比h大,那么吞并h的cluster,否则反向吞并 elif -1!=grid_object.clusterKey(): logging.debug("grid " + grid_object.key() + "has been cluster") grid_cluster_object=self.cluster_manager.getCluster(grid_object.clusterKey()) grid_h_cluster_object=self.cluster_manager.getCluster(grid_h_object.clusterKey()) if grid_cluster_object.size()>grid_h_cluster_object.size(): logging.info("cluster of grid "+grid_object.key()+" > cluster of grid"+grid_h_object.key()) self.cluster_manager.mergeCluster(grid_cluster_object.key(),grid_h_cluster_object.key()) else: logging.info("cluster of grid " + grid_object.key() + " <= cluster of grid" + grid_h_object.key()) self.cluster_manager.mergeCluster(grid_h_cluster_object.key(),grid_cluster_object.key()) #================================================= def __adjust_dense_neighbor_transitional(self,grid_object,grid_h_object): # 拿h的cluster grid_h_cluster_object = self.cluster_manager.getCluster(grid_h_object.clusterKey()) if -1==grid_object.clusterKey(): logging.info("grid "+grid_object.key()+"hasn't been clustered") grid_h_cluster_object.addGrid(grid_object) #把grid加到h的cluster里并判断此时h是不是outside,如果不是再把grid拿出来 if not grid_h_cluster_object.isOutsideGrid(grid_h_object): logging.info("grid "+grid_object.key()+" is not outside of cluster "+grid_h_cluster_object.key()+" if added to it") grid_h_cluster_object.delGrid(grid_object) # if grid_h_cluster_object.size()==0: # self.cluster_manager.delCluster(grid_h_cluster_object.key()) else: logging.info("grid " + grid_object.key() + "has been clustered") grid_cluster_object = self.cluster_manager.getCluster(grid_object.clusterKey()) #grid易主 if grid_cluster_object.size()>=grid_h_cluster_object.size(): logging.info("size of cluster of gird "+grid_object.key()+" is greater than grid "+grid_h_object.key()) grid_h_cluster_object.delGrid(grid_h_object) grid_cluster_object.addGrid(grid_h_object) if grid_h_cluster_object.size()==0: self.cluster_manager.delCluster(grid_h_cluster_object.key()) #================================================= def __adjust_dense(self,grid_object): #得到这个dense grid (暂且叫g)的所有neighboring grid,在所有neighbor中找出cluster最大的一个grid叫grid_h neighbors=self.grid_list.getNeighborGrids(grid_object.key()) if 0==len(neighbors): logging.debug("there is no neighbors in grid "+grid_object.key()) return -1 max_size=0 grid_h_object=None for item in neighbors: if item.clusterKey()!=-1: cluster=self.cluster_manager.getCluster(item.clusterKey()) if cluster.size()>max_size: max_size=cluster.size() grid_h_object=item #如果这个if触发,说明neighbor都是没有cluster的 if 0==max_size: logging.debug("neighbors of grid "+grid_object.key()+" is not clustered") return -2 #如果这个h是一个dense if DensityStatus.DENSE==grid_h_object.densityStatus(): logging.debug("neighbor h of grid "+grid_object.key()+" is Dense") self.__adjust_dense_neighbor_dense(grid_object,grid_h_object) #如果h是一个transitinal, elif DensityStatus.TRANSITIONAL==grid_h_object.densityStatus(): logging.debug("neighbor h of grid " + grid_object.key() + " is TRANSITIONAL") self.__adjust_dense_neighbor_transitional(grid_object,grid_h_object) return 0 #================================================= def __adjust_transitional(self,grid_object): neighbor_clusters=self.cluster_manager.getNeighborClusters(grid_object) #在neighbor的grid中找出一个cluster,这个cluster最大且当g加入以后g是outside the_ret_cluster_key=0 the_ret_cluster_size=0 for cluster in neighbor_clusters: the_ret_cluster_key=-1 the_ret_cluster_size=0 if cluster.size()>the_ret_cluster_size: if cluster.isOutsideIfAdd(grid_object): the_ret_cluster_size=cluster.size() the_ret_cluster_key=cluster.key() #for循环结束就能找到了,当然也可能没有 if -1!=the_ret_cluster_key and 0!=the_ret_cluster_size: if grid_object.clusterKey() != -1: cluster = self.cluster_manager.getCluster(grid_object.clusterKey()) cluster.delGrid(grid_object) if cluster.size() == 0: self.cluster_manager.delCluster(cluster.key()) target=self.cluster_manager.getCluster(the_ret_cluster_key) target.addGrid(grid_object) #================================================= #================================================= #================================================= def __initial_clustring(self): #把所有的dense的grid设置为单独的cluster dense_grids=self.grid_list.getDenseGrids() for grid in dense_grids: self.cluster_manager.addNewCluster(grid) stop_flag=0 #为0则不停止,为1停止,即一遍循环没有任何class被修改后标记为1 #在grid_list中找所有是dense的C_Vector,把他们的label按0开始各自设置,并在clusters里记录,其他的非dense的grid设置成no_class while 0==stop_flag: #标记类簇被改变,若有类簇发生改变则++ change_flag=0 #遍历clusters,拿到每个clusters的set all_clusters=self.cluster_manager.getAllCluster() keys=list(all_clusters.keys()) for k in keys: cluster=None if k in all_clusters: cluster=all_clusters[k] else: continue change_flag+=self.__initial_clustering_neighbors(cluster) #while停止 if 0==change_flag: stop_flag=1 def __initial_clustering_neighbors(self,cluster): change_flag=0 # 找到属于outside的grid outside_grids = cluster.getOutsideGrids() for outside_grid in outside_grids: # 对属于outside的grid,获取它的neighboring 的grid neighbor_grids = self.grid_list.getNeighborGrids(outside_grid.key()) for neighbor_grid in neighbor_grids: # 若outside的grid所在cluster的尺度大于neighboring的grid,则吞并neighboring的cluster,同时change_flag++ try: # print(neighbor_grid) neighbor_cluster = self.cluster_manager.getCluster(neighbor_grid.clusterKey()) if cluster.key() == neighbor_cluster.key(): # 如果两个key一样,则不需要执行 continue if cluster.size() > neighbor_cluster.size(): self.cluster_manager.mergeCluster(cluster.key(), neighbor_cluster.key()) change_flag += 1 return change_flag else: self.cluster_manager.mergeCluster(neighbor_cluster.key(), cluster.key()) change_flag += 1 return change_flag except KeyError: logging.debug("except KeyError") # 否则反向吞并,同时change_flag++ if neighbor_grid.clusterKey() != -1: print("your program are being in trouble!!!!!") exit() if neighbor_grid.densityStatus() == DensityStatus.TRANSITIONAL: cluster.addGrid(neighbor_grid) change_flag += 1 return change_flag def __adjust_clustring(self): #=======(这一步论文里没讲清楚,自己加上去的)============把所有的dense的grid设置为单独的cluster dense_grids=self.grid_list.getDenseGrids() for grid in dense_grids: if grid.isNotClustered() and grid.densityStatus()==DensityStatus.DENSE: logging.debug("dense grid "+grid.key()+" is of no cluster") self.cluster_manager.addNewCluster(grid) #得到一个已经改变过的change_flag==1的数组 change_grids=self.grid_list.getChangeGrids() #遍历这个数组,处理每一个grid的C_Vector for grid_object in change_grids: # 处理sparse if DensityStatus.SPARSE==grid_object.densityStatus(): logging.debug(grid_object.key()+" is being define as SPARSE") self.__adjust_sparse(grid_object) #处理dense if DensityStatus.DENSE==grid_object.densityStatus(): logging.debug(grid_object.key() + "is being define as DENSE to process") self.__adjust_dense(grid_object) #处理transitional if DensityStatus.TRANSITIONAL==grid_object.densityStatus(): logging.debug(grid_object.key() + "is being define as TRANSITIONAL") self.__adjust_transitional(grid_object) # 进入sporadic删除判定逻辑,处理所有grid def judgeAndremoveSporadic(self, current_time): grids=self.grid_list.getSparseGrids() for grid_object in grids: if SparseStatus.TODELETE == grid_object.sparseStatus(): if grid_object.clusterKey()!=-1: cluster=self.cluster_manager.getCluster(grid_object.clusterKey()) cluster.delGrid(grid_object) if cluster.size() == 0: self.cluster_manager.delCluster(cluster.key()) self.grid_list.delGrid(grid_object.key(), current_time) elif SparseStatus.TEMP == grid_object.sparseStatus() or SparseStatus.NORMAL == grid_object.sparseStatus(): # 判断s1和s2 if grid_object.densityThreshold(current_time) > grid_object.densityWithTime( current_time) and current_time >= (1 + Helper().beta) * grid_object.time_remove(): # 符合s1,s2 放给TODELETE grid_object.setSparseStatus(SparseStatus.TODELETE) else: # 回NORMAL grid_object.setSparseStatus(SparseStatus.NORMAL) def do_DStream(self,rawData): self.tc+=1 #得到key值后,我们将数据点打到相应的grid中,然后更新其信息 logging.info("add New Data") self.grid_list.addNewData(rawData,self.tc) #grid_key=Helper.getKeyFromRawData(rawData) #判断grid_list里面有没有对应key,没有先添加 #if not self.grid_list.has_key(grid_key): #1、创建新的C_Vector; #2、加入grid_list 里; #更新grid的信息 #if not 0==self.__refreshGrid(grid_key,rawData): # print("key 不存在,__refreshGrid调用失败\n") #TODO:首次到达gap if self.tc == self.gap: logging.debug("touch the gap first time") self.__initial_clustring() elif self.tc%self.gap == 0: logging.debug("TOUCH THE GAP") #判断sporadic的状态并删除符合条件的grid logging.debug("judgeAndremoveSporadic") self.judgeAndremoveSporadic(self.tc) logging.debug("__adjust_clustring") self.__adjust_clustring() #清空change为0 logging.debug("clearChangeFlag") self.grid_list.clearChangeFlag()
def __init__(self): ClusterManager.init() self.Operator = Operator() self.RecoveryManager = RecoveryManager()
def __init__(self): self.ipmi_module = IPMIManager() self.cluster_list = ClusterManager.getClusterList() self.config = ConfigParser.RawConfigParser() self.config.read('hass.conf') self.port = int(self.config.get("detection", "polling_port"))
def test_unique_tokens_generic(self): for pattern in TestTemplating.templatize.get_patterns(): cluster = ClusterManager.get_cluster(pattern) self.assertTrue( cluster.consensus(Artifact.get_unique_tokens, Util.token_key_comparator))
def delete_cluster(cluster_id): ClusterManager.deleteNode(cluster_id, HOST, write_DB=False) Postprocess.deleteInstance()
from Templatize import Templatize import sys import os """ cd Opportunity/ object.properties """ #Opportunity,Lead /Users/cghai/Documents/Code/BaselineTemplating/ActiveMDS VO.xml.xml,EO.xml.xml templatize = Templatize(debug=True, objects=sys.argv[1].split(','), mds_path=sys.argv[2], patterns=sys.argv[3].split(',')) root_dir = os.getcwd() for pattern in templatize.patterns: os.chdir(root_dir) cluster = ClusterManager.get_cluster(pattern) cluster.get_template() pattern_dir = cluster.get_name().replace('.', '_') artifacts = cluster.get_artifacts() template = artifacts[0].get_template() template.write('SmartBaseline.xml') for artifact in cluster.get_artifacts(): tokens = artifact.get_unique_tokens() obj_dir = root_dir + '/' + artifact.get_obj() os.chdir(obj_dir) file_name = 'object.properties' f = open(file_name, 'a') for token in sorted(tokens.keys()): f.write(token) f.write("=") f.write(tokens[token])
def updateAllCluster(self): try: #print("HASS updateAllCluster") ClusterManager.updateAllCluster() except Exception as e: logging.error("HASS--updateAllCluster fail :" + str(e))
for k, v in kv.items(): collection.append({ "key": k, "value": v }) resp = { "collection": collection } else: resp = { "message": "Invalid op" } consumer.send_json(resp) if __name__ == "__main__": cm = ClusterManager() num_server = 1 if len(sys.argv) > 1: num_server = int(sys.argv[1]) print(f"num_server={num_server}") # Listen to add node notifications Process(target=pullNotifications, args=(cm, 1999,)).start() for node_id in range(num_server): server_port = 2000 + node_id cm.registerNode(server_port) print(f"Starting a server at:{server_port}...") Process(target=server, args=(server_port,)).start()
class ApnLa: def __init__(self, jsonFile, loadModified, costType): self.rdp = Rdp(jsonFile, loadModified) clusterList = self.rdp.clusterlist updateT = self.rdp.updateT controlConflicts = self.rdp.controlConflicts tInvariants = self.rdp.tInvariants costManager = self.costCreator(costType) self.clusterManager = ClusterManager(clusterList, updateT, controlConflicts, tInvariants, costManager) # Transition firing sequence. Uses clusterManager to get transition to fire, updates marking vector in rdp # and then updates all necesary information in cluster manager. def fireNext(self): enabledT = self.rdp.calcularSensibilizadas() fireTransition = self.clusterManager.getFireTransition(enabledT) cost = self.rdp.fire(fireTransition) self.clusterManager.updateCost(fireTransition, cost) self.clusterManager.setClusterFiredTransition(fireTransition) self.clusterManager.setControlClusterFiredTransition( fireTransition) # TODO self.clusterManager.updateIfNecessary(fireTransition) return def switcharoo(self): # self.rdp.costVector[0] = 50 self.rdp.costVector[6] = 50 self.rdp.costVector[4] = 25 print(self.rdp.costVector) def printClusters(self): print("<br>CLUSTERS:<br>") print("&emspRegular clusters") for cluster in self.clusterManager.clusters: print('&emsp*', cluster.transitionList) if len(self.clusterManager.controlClusters) > 0: print("&emspControl clusters") for cluster in self.clusterManager.controlClusters: print('&emsp&emsp*', cluster.transitionList) def getClusterProbs(self): probs = [] for cluster in self.clusterManager.clusters: if cluster.LA is not None: probs.append(cluster.LA.probabilityVector.tolist()) for cluster in self.clusterManager.controlClusters: probs.append(cluster.LA.probabilityVector.tolist()) return probs def getClusterTransitions(self): labels = [] for cluster in self.clusterManager.clusters: if cluster.LA is not None: labels.append(cluster.transitionList) for cluster in self.clusterManager.controlClusters: labels.append(cluster.transitionList) return labels def costCreator(self, costType): if costType == "inv": costManager = invcostManager(self.rdp.tInvariants, self.rdp) elif costType == "simp": costManager = simpcostManager(self.rdp) else: costManager = simpcostManager(self.rdp) return costManager