def reduce(self, uid, values): values = [eval(text_dict) for text_dict in values] c = Cluster() c.uid = uid c_total = len(values) sqdist = 0.0 # set cluster center to sum of members for doc in values: for tokenid in doc: if c.tfidf.has_key(tokenid): c.tfidf[tokenid] += doc[tokenid] else: c.tfidf[tokenid] = doc[tokenid] # set cluster center, currently the sum, to the mean for tokenid in c.tfidf: c.tfidf[tokenid] = c.tfidf[tokenid] / float(c_total) # set sqdist to the squared sum of deviations from mean for doc in values: sqdist += MathUtil.compute_distance(c.tfidf, doc, squared=True) # Output the cluster center into file: clusteri self.emit("cluster" + str(c.uid), str(c)) # Output the within distance into file: distancei self.emit("distance" + str(c.uid), str(c.uid) + "|" + str(sqdist))
def reduce(self, uid, values): c = Cluster() c.uid = uid sqdist = 0.0 # TODO: update the cluster center instances = [] for value in values: doc = Document(value) instances.append(doc) for token in doc.tfidf.keys(): bool = token in c.tfidf.keys() if bool: c.tfidf[token] += doc.tfidf[token] else: c.tfidf[token] = doc.tfidf[token] size = float(len(values)) for token in c.tfidf.keys(): c.tfidf[token] = c.tfidf[token]/size #compute the distance for instance in instances: sqdist += MathUtil.compute_distance(map1 = c.tfidf, map2 = instance.tfidf) # Output the cluster center into file: clusteri self.emit("cluster" + str(c.uid), str(c)) # Output the within distance into file: distancei self.emit("distance" + str(c.uid), str(c.uid) + "|" + str(sqdist))
def construct(options, system, ruby_system): if (buildEnv['PROTOCOL'] != 'GPU_VIPER' or buildEnv['PROTOCOL'] != 'GPU_VIPER_Region' or buildEnv['PROTOCOL'] != 'GPU_VIPER_Baseline'): panic("This script requires VIPER based protocols \ to be built.") cpu_sequencers = [] cpuCluster = None cpuCluster = Cluster(name="CPU Cluster", extBW = 8, intBW=8) # 16 GB/s for i in xrange((options.num_cpus + 1) / 2): cp_cntrl = CPCntrl() cp_cntrl.create(options, ruby_system, system) # Connect the CP controllers to the ruby network cp_cntrl.requestFromCore = ruby_system.network.slave cp_cntrl.responseFromCore = ruby_system.network.slave cp_cntrl.unblockFromCore = ruby_system.network.slave cp_cntrl.probeToCore = ruby_system.network.master cp_cntrl.responseToCore = ruby_system.network.master exec("system.cp_cntrl%d = cp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1]) cpuCluster.add(cp_cntrl) return cpu_sequencers, cpuCluster
def __init__(self, tf, k, capacity): self.ccnt = k # Total number of clusters. self.capacity = capacity self.tf = open(tf) self.itemset = set([]) # keep uniqueness. n=0 # number of occurrences. cnt =0 # number of trans. for line in self.tf: items = line.strip().split(' ') [self.itemset.add(i) for i in items] n += len(items) cnt +=1 n /= cnt # Create k clusters and # Create list for all items appear in the trans. root = Cluster(0) root.leaf = False self.clusters = [root] self.itemset = list(self.itemset) for i in range(2): c = Cluster(i) #c.rand_init(self.itemset,int(n) ) self.clusters.append(c) c.pprint() #self.clusters[0].add_trans(['a','b','c']) #self.clusters[1].add_trans(['x','y','z']) self.members = [] exit
def testCluster(self): cluster_test = Cluster([point]) self.assertEquals(cluster_test.dimension, 2) self.assertFalse(cluster_test.converge) np.testing.assert_array_equal(cluster_test.centroid, np.array([2, 2])) cluster_test.update_cluster(list_points) self.assertEquals(cluster_test.dimension, 2) self.assertTrue(cluster_test.converge) np.testing.assert_array_equal(cluster_test.centroid, np.array([2, 2]))
def checkDataPoint(dataPoint): for cluster in clusterDb.clusters: if cluster.checkDataPoint(dataPoint): print 'Add point to: ' + cluster.toString() cluster.addDataPoint(dataPoint) print 'and gives: ' + cluster.toString() return newCluster = Cluster() newCluster.addDataPoint(dataPoint) clusterDb.addCluster(newCluster) print 'Made new cluser: ' + newCluster.toString()
def test_cluster(self): cluster_test = Cluster([point], 1) self.assertEquals(cluster_test.dimension, 2) self.assertFalse(cluster_test.converge) np.testing.assert_array_equal(cluster_test.mean, np.array([2, 2])) np.testing.assert_array_equal(cluster_test.std, np.array([1, 1])) self.assertEquals(cluster_test.cluster_probability, 1) cluster_test.update_cluster(listPoints, 4) self.assertEquals(cluster_test.dimension, 2) self.assertTrue(cluster_test.converge) np.testing.assert_array_equal(cluster_test.mean, np.array([2, 2])) self.assertEquals(cluster_test.std[0] - cluster_test.std[1], 0) self.assertEquals(cluster_test.cluster_probability, 1)
def kmeansClustering(cluster_list, k, iterations, shuffle = True): """ Compute the k-means clustering of a set of clusters (reads/kmers) Note: the function may not mutate cluster_list Input: List of clusters, k number of clusters, iterations, select initial clusters: randomly or by size? Output: List of clusters. """ kclusters = [] # this list to store k clusters to compare with (non-mutable) centroids = [] # this list to store the initial k centroids (average stats vectors) if shuffle: # shuffle cluster list random.shuffle(cluster_list) else: # sort by size cluster_list.sort(key = lambda cluster: cluster.getSize(), reverse = True) # k initial clusters to define initial centroids for cluster in cluster_list[:k]: kclusters.append(cluster.copy()) centroids.append(cluster.getAvgAbundance()) for iteration in range(iterations): clusters = [] # initialize new empty cluster objects at the centroids for idx in range(k): cluster = Cluster([]) cluster.avg_abundance_vectors = list(centroids[idx]) clusters.append(cluster) # for every cluster in cluster_list for num in range(len(cluster_list)): best = (float('inf'), -1) # compare distance to every centroid at kclusters for idx in range(k): temp = cluster_list[num].distance(kclusters[idx]) if temp < best[0]: best = (temp, idx) # merge cluster to best centroid in list of mutable clusters clusters[best[1]].mergeClusters(cluster_list[num]) # make a copy of re-computed centroids: kclusters and centroids. for idx in range(k): kclusters[idx] = clusters[idx].copy() centroids[idx] = (clusters[idx].getAvgAbundance()) return kclusters
def load_cluster(self, path, initial=False): """ Load the cluster centers from hdfs path. @param path """ clusters = [] for line in HDFSUtil.read_lines(path, hadoop_prefix=HADOOP_PREFIX): if not initial: if line.startswith("cluster"): line = line.split("\t", 1)[1] else: continue c = Cluster() c.read(line) clusters.append(c) return clusters
def main(self): if self.iteration == 1: path = self.kmeans_hdfs_path + "/cluster0/cluster0.txt" else: path = self.kmeans_hdfs_path + "/output/cluster" + str(self.iteration - 1) + "/part-00000" for line in HDFSUtil.read_lines(path, hadoop_prefix=self.hadoop_prefix): if self.iteration > 1: if line.startswith("cluster"): line = line.split("\t", 1)[1] else: continue c = Cluster() c.read(line) self.clusters.append(c) data = self.read_input(sys.stdin) for line in data: self.map(line)
def printSystemInfo(prefix): """Print system information to stdout. Print prefix first.""" if prefix: Utils.Print(str(prefix)) clientVersion=Cluster.getClientVersion() Utils.Print("UTC time: %s" % str(datetime.utcnow())) Utils.Print("EOS Client version: %s" % (clientVersion)) Utils.Print("Processor: %s" % (platform.processor())) Utils.Print("OS name: %s" % (platform.platform()))
def __init__(self, cluster_id, customers, optimal_solution = None, sum_cargo = None, radius = None, center = None, calculate = True): self.customers = customers self.sum_cargo = sum_cargo self.optimal_solution = optimal_solution self.cluster_id = cluster_id self.radius = radius self.center = center self.cluster = Cluster(customers) if calculate: self.custom_solution()
class ClusterSolution(): def __init__(self, cluster_id, customers, optimal_solution = None, sum_cargo = None, radius = None, center = None, calculate = True): self.customers = customers self.sum_cargo = sum_cargo self.optimal_solution = optimal_solution self.cluster_id = cluster_id self.radius = radius self.center = center self.cluster = Cluster(customers) if calculate: self.custom_solution() def custom_solution(self): self.optimal_solution = self.cluster.get_solution(self.customers) self.sum_cargo = self.cluster.get_cargo() self.radius = self.cluster.get_radius() self.center = self.cluster.get_center() def next_to_visit_ids(self, customer): """Given a customer ID, returns the customer ID of the next customer on the optimal solution""" index = self.optimal_solution.route.index( filter( lambda c: c.number == customer, self.optimal_solution.route )[0] ) return self.optimal_solution.route[( index + 1 ) % len( self.customers )].number
def zookeeper(restart, start, stop, force_stop, clean, zk_servers, wait_before_start, wait_before_kill): cluster = Cluster() if(restart or stop): logging.debug("Shutting down Zookeeper") cluster.shutdownZookeeper() if(force_stop): logging.debug("Waiting for "+str(wait_before_kill)+" seconds") sleep(wait_before_kill) logging.debug("Force Killing Zookeeper") cluster.killZookeeper() if(clean): logging.debug("Cleaning Zookeeper") cluster.cleanZookeeper() if(restart or start): logging.debug("Waiting for "+str(wait_before_start)+" seconds") sleep(wait_before_start) logging.debug("Starting Zookeeper") cluster.startZookeeper() click.echo(cluster.getReport())
def getAmenities(self): """Extract the required map features from the database for each map square and store the data in the 'amenities' dictionary. An index of the street names is stored in self.streetIndex dictionary. FIXME: This is very inefficient - it queries the database for the bounding box and each required feature in turn. It sould be more efficient to get all of the data in the bounding box then query that, but I haven't done it. 01oct2009 GJ ORIGINAL VERSION 04oct2009 GJ Added use of expandWhereClause to allow grouping of different key values under the same heading in the features page. 11oct2009 GJ Added support for areas (polygons) as well as nodes. """ # Extract the data for each 1km square. if (self.debug): print "getAmenities()" c0 = self.c0 # lon, lat streetList = {} streetIndexSorted = {} print c0 # Get all the features and then create clusters === if is_true(self.preferences_list['clusters']): print "........... Trying out something .........." tilesize = self.preferences_list['tilesize'] map_width = self.map_size_x map_height = self.preferences_list['map_size_y'] min_lon = c0.x min_lat = c0.y max_lon = c0.x + (tilesize * map_width) max_lat = c0.y + (tilesize * map_height) # Experimental cluster markers # tolerance to become a cluster should change according to the zooming factor tolerance = 20 * self.map_size_x # k-means first iteration clusters = {} if HAS_MAPNIK2: bbox = mapnik.Box2d(min_lon, min_lat, max_lon, max_lat) else: bbox = mapnik.Envelope(min_lon, min_lat, max_lon, max_lat) for featureStr in self.features: title,wc = self.expandWhereClause(featureStr) print "::::", title, wc feature = title if self.debug: print "Extracting feature %s using %s." %\ (feature,featureStr) pois = self.getBBContents(bbox,wc) # poi[6]: lat # poi[7]: lon print "Current Contents for ", featureStr, pois for poi in pois: if not len(self.clusters) > 0: print "POI:", poi c = Cluster(feature) c.add_poi(poi) self.clusters.append(c) print "Created the First cluster: ", c.centroid else: # Look for the custer where the POI should be inserted # by noticing the difference with that cluster's centroid # If the distance to that cluster's centroid is less than # the tolerance it will be inserted there. # This should work like a non-optimal first iteration of k-means CLUSTER_FOUND = False for (scounter, cluster) in enumerate(self.clusters): distance = cluster.distance_from_poi(poi) print "This cluster", cluster.pois print distance if distance < tolerance: # Add POI to cluster print "--- POI should get in this cluster" cluster.add_poi(poi) CLUSTER_FOUND = True print "BREAK!!!" break if not CLUSTER_FOUND: print "_____ Could not find a suited cluster _____" # Create a new cluster print "------ Creating a new cluster for this poi" new_cluster = Cluster(feature) new_cluster.add_poi(poi) self.clusters.append(new_cluster) for kluster in self.clusters: print len(kluster.pois), vars(kluster) print "............................................." #=========================================================== for tx in range(0,self.map_size_x): minx = c0.x + self.preferences_list['tilesize'] * tx for ty in range(0,self.preferences_list['map_size_y']): # sys.stdout.write("%s" % self.cellLabel(tx,ty)) # sys.stdout.flush() print "%s " % self.cellLabel(tx,ty) miny = c0.y + self.preferences_list['tilesize'] * ty if HAS_MAPNIK2: bbox = mapnik.Box2d(minx, miny, minx + self.preferences_list['tilesize'], miny + self.preferences_list['tilesize']) else: bbox = mapnik.Envelope(minx, miny, minx + self.preferences_list['tilesize'], miny + self.preferences_list['tilesize']) fname = "image_%02d_%02d.png" % (tx,ty) #if self.debug: print bbox ######################################################## # Extract points of interest into amenities dictionary # ######################################################## for featureStr in self.features: title,wc = self.expandWhereClause(featureStr) # print "::::", title, wc feature = title if self.debug: print "Extracting feature %s using %s." %\ (feature,featureStr) # FIXME: This needs to be optimized for clustering pois = self.getBBContents(bbox,wc) for poi in pois: if feature in self.amenities: self.amenities[feature].append((tx,ty,poi)) else: self.amenities[feature]=[(tx,ty,poi)] # print self.amenities # print len(self.amenities) # print "__________________________________" ############################################## # Extract all of the streetnames in the cell # ############################################## bbStreets = self.getBBStreets(bbox) streetList[self.cellLabel(tx,ty)] = bbStreets # Render a high resolution tile of this cell. #fname = "%s/%s.png" % (self.outdir,self.cellLabel(tx,ty)) #self.drawTile(bbox,1000,1000,fname) ##################################################### # Sort the amenities list to remove duplicates # # The result is self.amenitiesSorted which is a # # dictionary of features. Each feature contains # # a dictionary of amenity names. Each amenity # # name entry contains a list of cell IDs as strings # # # ##################################################### for feature in self.amenities.keys(): # poi = (tx,ty,(id, name, operator)) # example: # (0, 0, (663528594, 'Tangerine', None, 'restaurant', None, None, 4546182.5512522003, -13628929.6157306)) for poi in self.amenities[feature]: cellId = self.cellLabel(poi[0],poi[1]) osm_id = poi[2][0] # 663528594 name = poi[2][1] # 'Tangerine' operator = poi[2][2] # Company that runs the place amenityVal = poi[2][3] # Type of amenity shopVal = poi[2][4] landuse = poi[2][5] if name == None: if amenityVal == None: if shopVal == None: if landuse == None: name = "Unidentified thing - osm_id=%d" % osm_id else: name = "Unnamed %s" % landuse else: name = "Unnamed %s" % shopVal else: name = "Unnamed %s" % amenityVal print "%s, %s, %s" % (feature, name, cellId) # Entertainment, Castro Theater, A1 if not feature in self.amenitiesSorted: print "creating dictionary for feature %s" % feature self.amenitiesSorted[feature]={} if name in self.amenitiesSorted[feature]: print "dictionary %s already exists in feature %s" % (name,feature) if not cellId in self.amenitiesSorted[feature][name]: print "appending cellid" self.amenitiesSorted[feature][name].append(cellId) else: print "skipping duplicate feature %s %s in %s\n" % (feature,name,cellId) else: print "adding cell id to list" self.amenitiesSorted[feature][name] = [cellId] print "self.amenitiesSorted=", self.amenitiesSorted[feature][name] ######################################################### # Sort the street index into a simple dictionary of # # street name mapped to a list of which cells contain # # parts of the street of that name # ######################################################### # In the following, streetList is a dictionary with keys which # are cell IDs - the contents of the dictionary is a list of all # of the streets in a given cell. # StreeetIndexSorted is a dictionary with streetnames as the keys # The contents is a list of all of the cells that contain a way # of the given name. # self.StreetIndex is a tidied up version of StreetIndexSorted - the # contents is a displayable string showing which cells contain the # given street name. # Now we need to sort the street index so that we have a schedule # of streets identifying which cells the street is in. for cell in streetList: for street in streetList[cell]: streetName = street[1] if streetName != 'None': if streetName in streetIndexSorted: # Avoid multiple entries of the same cell for a # given street name - if it exists in the list, # reject it, otherwise add it to the list.. try: i = streetIndexSorted[streetName].index(cell) if self.debug: print "Rejected Duplicate Entry ",\ streetName,cell,i except: streetIndexSorted[streetName].append(cell) else: streetIndexSorted[streetName] = [cell] streets = streetIndexSorted.keys() streets.sort() for street in streets: cellstr="" first = True cells = streetIndexSorted[street] cells.sort() for cell in cells: if first==True: cellstr = "%s" % cell first=False else: cellstr = "%s, %s" % (cellstr,cell) self.streetIndex[street]=cellstr
"ERROR: eosio-blocklog didn't return block output") block_num = block['block_num'] assert block_num == expected_block_num expected_block_num += 1 Print("Block_log contiguous from block number %d to %d" % (firstBlockNum, expected_block_num - 1)) appArgs = AppArgs() args = TestHelper.parse_args({ "--dump-error-details", "--keep-logs", "-v", "--leave-running", "--clean-run" }) Utils.Debug = args.v pnodes = 2 cluster = Cluster(walletd=True) dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontKill = args.leave_running prodCount = 2 killAll = args.clean_run walletPort = TestHelper.DEFAULT_WALLET_PORT totalNodes = pnodes + 1 walletMgr = WalletMgr(True, port=walletPort) testSuccessful = False killEosInstances = not dontKill killWallet = not dontKill WalletdName = Utils.EosWalletName ClientName = "cleos"
def create_system(options, full_system, system, dma_devices, ruby_system): if buildEnv['PROTOCOL'] != 'GPU_VIPER': panic("This script requires the GPU_VIPER protocol to be built.") cpu_sequencers = [] # # The ruby network creation expects the list of nodes in the system to be # consistent with the NetDest list. Therefore the l1 controller nodes # must be listed before the directory nodes and directory nodes before # dma nodes, etc. # cp_cntrl_nodes = [] tcp_cntrl_nodes = [] sqc_cntrl_nodes = [] tcc_cntrl_nodes = [] dir_cntrl_nodes = [] l3_cntrl_nodes = [] # # Must create the individual controllers before the network to ensure the # controller constructors are called before the network constructor # # For an odd number of CPUs, still create the right number of controllers TCC_bits = int(math.log(options.num_tccs, 2)) # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu # Clusters crossbar_bw = None mainCluster = None if hasattr(options, 'bw_scalor') and options.bw_scalor > 0: #Assuming a 2GHz clock crossbar_bw = 16 * options.num_compute_units * options.bw_scalor mainCluster = Cluster(intBW=crossbar_bw) else: mainCluster = Cluster(intBW=8) # 16 GB/s for i in xrange(options.num_dirs): dir_cntrl = DirCntrl(noTCCdir = True, TCC_select_num_bits = TCC_bits) dir_cntrl.create(options, ruby_system, system) dir_cntrl.number_of_TBEs = options.num_tbes dir_cntrl.useL3OnWT = options.use_L3_on_WT # the number_of_TBEs is inclusive of TBEs below # Connect the Directory controller to the ruby network dir_cntrl.requestFromCores = MessageBuffer(ordered = True) dir_cntrl.requestFromCores.slave = ruby_system.network.master dir_cntrl.responseFromCores = MessageBuffer() dir_cntrl.responseFromCores.slave = ruby_system.network.master dir_cntrl.unblockFromCores = MessageBuffer() dir_cntrl.unblockFromCores.slave = ruby_system.network.master dir_cntrl.probeToCore = MessageBuffer() dir_cntrl.probeToCore.master = ruby_system.network.slave dir_cntrl.responseToCore = MessageBuffer() dir_cntrl.responseToCore.master = ruby_system.network.slave dir_cntrl.triggerQueue = MessageBuffer(ordered = True) dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True) dir_cntrl.responseFromMemory = MessageBuffer() exec("ruby_system.dir_cntrl%d = dir_cntrl" % i) dir_cntrl_nodes.append(dir_cntrl) mainCluster.add(dir_cntrl) cpuCluster = None if hasattr(options, 'bw_scalor') and options.bw_scalor > 0: cpuCluster = Cluster(extBW = crossbar_bw, intBW = crossbar_bw) else: cpuCluster = Cluster(extBW = 8, intBW = 8) # 16 GB/s for i in xrange((options.num_cpus + 1) / 2): cp_cntrl = CPCntrl() cp_cntrl.create(options, ruby_system, system) exec("ruby_system.cp_cntrl%d = cp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1]) # Connect the CP controllers and the network cp_cntrl.requestFromCore = MessageBuffer() cp_cntrl.requestFromCore.master = ruby_system.network.slave cp_cntrl.responseFromCore = MessageBuffer() cp_cntrl.responseFromCore.master = ruby_system.network.slave cp_cntrl.unblockFromCore = MessageBuffer() cp_cntrl.unblockFromCore.master = ruby_system.network.slave cp_cntrl.probeToCore = MessageBuffer() cp_cntrl.probeToCore.slave = ruby_system.network.master cp_cntrl.responseToCore = MessageBuffer() cp_cntrl.responseToCore.slave = ruby_system.network.master cp_cntrl.mandatoryQueue = MessageBuffer() cp_cntrl.triggerQueue = MessageBuffer(ordered = True) cpuCluster.add(cp_cntrl) gpuCluster = None if hasattr(options, 'bw_scalor') and options.bw_scalor > 0: gpuCluster = Cluster(extBW = crossbar_bw, intBW = crossbar_bw) else: gpuCluster = Cluster(extBW = 8, intBW = 8) # 16 GB/s for i in xrange(options.num_compute_units): tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits, issue_latency = 1, number_of_TBEs = 2560) # TBEs set to max outstanding requests tcp_cntrl.create(options, ruby_system, system) tcp_cntrl.WB = options.WB_L1 tcp_cntrl.disableL1 = options.noL1 tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(tcp_cntrl.coalescer) tcp_cntrl_nodes.append(tcp_cntrl) # Connect the TCP controller to the ruby network tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcp_cntrl.requestFromTCP.master = ruby_system.network.slave tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True) tcp_cntrl.responseFromTCP.master = ruby_system.network.slave tcp_cntrl.unblockFromCore = MessageBuffer() tcp_cntrl.unblockFromCore.master = ruby_system.network.slave tcp_cntrl.probeToTCP = MessageBuffer(ordered = True) tcp_cntrl.probeToTCP.slave = ruby_system.network.master tcp_cntrl.responseToTCP = MessageBuffer(ordered = True) tcp_cntrl.responseToTCP.slave = ruby_system.network.master tcp_cntrl.mandatoryQueue = MessageBuffer() gpuCluster.add(tcp_cntrl) for i in xrange(options.num_sqc): sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits) sqc_cntrl.create(options, ruby_system, system) exec("ruby_system.sqc_cntrl%d = sqc_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(sqc_cntrl.sequencer) # Connect the SQC controller to the ruby network sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True) sqc_cntrl.requestFromSQC.master = ruby_system.network.slave sqc_cntrl.probeToSQC = MessageBuffer(ordered = True) sqc_cntrl.probeToSQC.slave = ruby_system.network.master sqc_cntrl.responseToSQC = MessageBuffer(ordered = True) sqc_cntrl.responseToSQC.slave = ruby_system.network.master sqc_cntrl.mandatoryQueue = MessageBuffer() # SQC also in GPU cluster gpuCluster.add(sqc_cntrl) for i in xrange(options.num_cp): tcp_ID = options.num_compute_units + i sqc_ID = options.num_sqc + i tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits, issue_latency = 1, number_of_TBEs = 2560) # TBEs set to max outstanding requests tcp_cntrl.createCP(options, ruby_system, system) tcp_cntrl.WB = options.WB_L1 tcp_cntrl.disableL1 = options.noL1 tcp_cntrl.L1cache.tagAccessLatency = options.TCP_latency tcp_cntrl.L1cache.dataAccessLatency = options.TCP_latency exec("ruby_system.tcp_cntrl%d = tcp_cntrl" % tcp_ID) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(tcp_cntrl.sequencer) tcp_cntrl_nodes.append(tcp_cntrl) # Connect the CP (TCP) controllers to the ruby network tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcp_cntrl.requestFromTCP.master = ruby_system.network.slave tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True) tcp_cntrl.responseFromTCP.master = ruby_system.network.slave tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True) tcp_cntrl.unblockFromCore.master = ruby_system.network.slave tcp_cntrl.probeToTCP = MessageBuffer(ordered = True) tcp_cntrl.probeToTCP.slave = ruby_system.network.master tcp_cntrl.responseToTCP = MessageBuffer(ordered = True) tcp_cntrl.responseToTCP.slave = ruby_system.network.master tcp_cntrl.mandatoryQueue = MessageBuffer() gpuCluster.add(tcp_cntrl) sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits) sqc_cntrl.create(options, ruby_system, system) exec("ruby_system.sqc_cntrl%d = sqc_cntrl" % sqc_ID) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(sqc_cntrl.sequencer) # SQC also in GPU cluster gpuCluster.add(sqc_cntrl) for i in xrange(options.num_tccs): tcc_cntrl = TCCCntrl(l2_response_latency = options.TCC_latency) tcc_cntrl.create(options, ruby_system, system) tcc_cntrl.l2_request_latency = options.gpu_to_dir_latency tcc_cntrl.l2_response_latency = options.TCC_latency tcc_cntrl_nodes.append(tcc_cntrl) tcc_cntrl.WB = options.WB_L2 tcc_cntrl.number_of_TBEs = 2560 * options.num_compute_units # the number_of_TBEs is inclusive of TBEs below # Connect the TCC controllers to the ruby network tcc_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcc_cntrl.requestFromTCP.slave = ruby_system.network.master tcc_cntrl.responseToCore = MessageBuffer(ordered = True) tcc_cntrl.responseToCore.master = ruby_system.network.slave tcc_cntrl.probeFromNB = MessageBuffer() tcc_cntrl.probeFromNB.slave = ruby_system.network.master tcc_cntrl.responseFromNB = MessageBuffer() tcc_cntrl.responseFromNB.slave = ruby_system.network.master tcc_cntrl.requestToNB = MessageBuffer(ordered = True) tcc_cntrl.requestToNB.master = ruby_system.network.slave tcc_cntrl.responseToNB = MessageBuffer() tcc_cntrl.responseToNB.master = ruby_system.network.slave tcc_cntrl.unblockToNB = MessageBuffer() tcc_cntrl.unblockToNB.master = ruby_system.network.slave tcc_cntrl.triggerQueue = MessageBuffer(ordered = True) exec("ruby_system.tcc_cntrl%d = tcc_cntrl" % i) # connect all of the wire buffers between L3 and dirs up # TCC cntrls added to the GPU cluster gpuCluster.add(tcc_cntrl) # Assuming no DMA devices assert(len(dma_devices) == 0) # Add cpu/gpu clusters to main cluster mainCluster.add(cpuCluster) mainCluster.add(gpuCluster) ruby_system.network.number_of_virtual_networks = 10 return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
args = TestHelper.parse_args({ "--defproducera_prvt_key", "--dump-error-details", "--dont-launch", "--keep-logs", "-v", "--leave-running", "--clean-run", "--p2p-plugin" }) debug = args.v defproduceraPrvtKey = args.defproducera_prvt_key dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontLaunch = args.dont_launch dontKill = args.leave_running killAll = args.clean_run p2pPlugin = args.p2p_plugin Utils.Debug = debug cluster = Cluster(walletd=True, defproduceraPrvtKey=defproduceraPrvtKey) walletMgr = WalletMgr(True) testSuccessful = False killEosInstances = not dontKill killWallet = not dontKill WalletdName = Utils.DADAPPSWalletName ClientName = "cleos" timeout = .5 * 12 * 2 + 60 # time for finalization with 1 producer + 60 seconds padding Utils.setIrreversibleTimeout(timeout) try: TestHelper.printSystemInfo("BEGIN") cluster.setWalletMgr(walletMgr)
chainSyncStrategyStr = args.c debug = args.v total_nodes = pnodes killCount = args.kill_count if args.kill_count > 0 else 1 killSignal = args.kill_sig killLtnInstances = not args.leave_running dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs killAll = args.clean_run seed = 1 Utils.Debug = debug testSuccessful = False random.seed(seed) # Use a fixed seed for repeatability. cluster = Cluster(walletd=True) walletMgr = WalletMgr(True) try: TestHelper.printSystemInfo("BEGIN") cluster.setWalletMgr(walletMgr) cluster.setChainStrategy(chainSyncStrategyStr) cluster.setWalletMgr(walletMgr) cluster.killall(allInstances=killAll) cluster.cleanup() walletMgr.killall(allInstances=killAll) walletMgr.cleanup() Print(
info1=prodNodes[1].getInfo(exitOnError=True) headBlockNum=min(int(info0["head_block_num"]),int(info1["head_block_num"])) libNum=min(int(info0["last_irreversible_block_num"]), int(info1["last_irreversible_block_num"])) return (headBlockNum, libNum) args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run", "--wallet-port"}) Utils.Debug=args.v totalProducerNodes=2 totalNonProducerNodes=1 totalNodes=totalProducerNodes+totalNonProducerNodes maxActiveProducers=21 totalProducers=maxActiveProducers cluster=Cluster(walletd=True) dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running prodCount=args.prod_count killAll=args.clean_run walletPort=args.wallet_port walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill WalletdName=Utils.EosWalletName ClientName="cleos"
defproduceraPrvtKey=args.defproducera_prvt_key defproducerbPrvtKey=args.defproducerb_prvt_key dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontLaunch=args.dont_launch dontKill=args.leave_running prodCount=args.prod_count onlyBios=args.only_bios killAll=args.clean_run sanityTest=args.sanity_test p2pPlugin=args.p2p_plugin walletPort=args.wallet_port Utils.Debug=debug localTest=True if server == TestHelper.LOCAL_HOST else False cluster=Cluster(host=server, port=port, walletd=True, enableMongo=enableMongo, defproduceraPrvtKey=defproduceraPrvtKey, defproducerbPrvtKey=defproducerbPrvtKey) walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill dontBootstrap=sanityTest # intent is to limit the scope of the sanity test to just verifying that nodes can be started WalletdName=Utils.EosWalletName ClientName="cleon" timeout = .5 * 12 * 2 + 60 # time for finalization with 1 producer + 60 seconds padding Utils.setIrreversibleTimeout(timeout) try: TestHelper.printSystemInfo("BEGIN") cluster.setWalletMgr(walletMgr) Print("SERVER: %s" % (server))
# Parse command line arguments args = TestHelper.parse_args({ "-v", "--clean-run", "--dump-error-details", "--leave-running", "--keep-logs", "--alternate-version-labels-file" }) Utils.Debug = args.v killAll = args.clean_run dumpErrorDetails = args.dump_error_details dontKill = args.leave_running killscsInstances = not dontKill killWallet = not dontKill keepLogs = args.keep_logs alternateVersionLabelsFile = args.alternate_version_labels_file walletMgr = WalletMgr(True) cluster = Cluster(walletd=True) cluster.setWalletMgr(walletMgr) def restartNode(node: Node, nodeId, chainArg=None, addSwapFlags=None, nodscsPath=None): if not node.killed: node.kill(signal.SIGTERM) isRelaunchSuccess = node.relaunch(nodeId, chainArg, addSwapFlags=addSwapFlags, timeout=5, cachePopen=True,
class DataAnalysis: def __init__(self, path_): self.path = path_ self.data_object = None self.data = None self.forward_trips = {} self.backwards_trips = {} self.sort_ft = [] self.sort_bt = [] self.forward_split = {} self.backward_split = {} self.outliers = {} self.c_object = None def count_items(self, dict_): num = 0 for items in dict_: num += len(dict_[items]) return num def load_data(self, file_): """Loads sorted valid trips from DataProcess class""" self.data_object = RawDataManager(self.path) self.data_object.load_sorted_trips(file_) self.data = self.data_object.get_valid_trips() def get_trips(self, sequence): """finds trips in all valid trips that contain the desired sequence of sensors where sequence = [sensor1, sensor2....]""" self.forward_trips = self.data_object.check_seq(sequence) print "Foward Trips:", len(self.forward_trips) reverse = sequence reverse.reverse() self.backwards_trips = self.data_object.check_seq(reverse) print "Backward Trips:", len(self.backwards_trips) def find_times(self, timelist): """finds the start time and end time in a list of sensor data, where some sensors may have two readings. In this case the first sensor is taken""" if type(timelist[0][1]) == tuple: start_time = sorted(timelist[0], key=lambda x: x[-1])[0][0][0] else: start_time = timelist[0][0][0] if type(timelist[-1][1]) == tuple: end_time = sorted(timelist[-1], key=lambda x: x[-1])[0][0][0] else: end_time = timelist[-1][0][0] return (datetime.utcfromtimestamp(start_time), datetime.utcfromtimestamp(end_time)) def list_times(self, group, type0_=None): """for a group of 50 trips, this finds the start time, end time and total time for each trip""" start_times = [] end_times = [] for trip in group: (start, end) = self.find_times(trip[1]) if type0_ == 'time': start = start.time() end = end.time() start_times.append(start) end_times.append(end) return start_times, end_times def diff_time(self, time1, time2): """finds the difference in minutes between two timestamps""" start = datetime.utcfromtimestamp(time1) end = datetime.utcfromtimestamp(time2) total_time = end - start return total_time.total_seconds() / 60. def sort_trips(self, type_=None): """sorts trips containing desired sequnce by day and time or just time""" if type_ == 'day' or type_ == None: self.sort_ft = sorted(self.forward_trips.items(), key=lambda x: x[0][2]) self.sort_bt = sorted(self.backwards_trips.items(), key=lambda x: x[0][2]) if type_ == 'time': self.sort_ft = sorted( self.forward_trips.items(), key=lambda x: datetime.utcfromtimestamp(x[0][2]).time()) self.sort_bt = sorted( self.backwards_trips.items(), key=lambda x: datetime.utcfromtimestamp(x[0][2]).time()) def split_trips(self, group_number, overlap=None): """splits sorted trips into groups of fifty, if overlap it set to a number, groups will overlap by that number""" tripi = 0 group = 1 while tripi < len(self.sort_ft): i = 0 trips = [] while i < group_number: if tripi < len(self.sort_ft): trips.append(self.sort_ft[tripi]) tripi = tripi + 1 i = i + 1 else: break if len(trips) < 10: old_trips = self.forward_split['Group ' + str(group - 1)] new_trips = old_trips + trips self.forward_split['Group ' + str(group - 1)] = new_trips break self.forward_split['Group ' + str(group)] = trips group = group + 1 if overlap: if tripi < len(self.sort_ft): if overlap < group_number: tripi = tripi - overlap else: raise ValueError, 'overlap value too large' tripi = 0 group = 1 while tripi < len(self.sort_bt): i = 0 trips = [] while i < group_number: if tripi < len(self.sort_bt): trips.append(self.sort_bt[tripi]) tripi = tripi + 1 i = i + 1 else: break if len(trips) < 10: old_trips = self.forward_split['Group ' + str(group - 1)] new_trips = old_trips + trips self.backward_split['Group ' + str(group - 1)] = new_trips break self.backward_split['Group ' + str(group)] = trips group = group + 1 if overlap: if tripi < len(self.sort_bt): if overlap < group_number: tripi = tripi - overlap else: raise ValueError, 'overlap value too large' def get_avg_stdv(self, timelist): """returns average and stdv of a list of values (timelist)""" sum_ = 0 var = 0. for times in timelist: sum_ = sum_ + times average = sum_ / len(timelist) for times in timelist: var = var + (times - average)**2 var = var / len(timelist) std = np.sqrt(var) return average, std def get_time_value(self, info): """returns time of sensor detection from sensor list""" if type(info[1]) == tuple: time = sorted(info, key=lambda x: x[-1])[0][0][0] else: time = info[0][0] return time def find_R(self, X, Y): """finds the R value from two lists of link times""" if len(X) != len(Y): print X, Y raise ValueError, 'unequal length' sum_ = error = residual = 0. avgx, stdvx = self.get_avg_stdv(X) avgy, stdvy = self.get_avg_stdv(Y) n = float(len(X)) for x, y in zip(X, Y): xin = (x - avgx) / stdvx yin = (y - avgy) / stdvy sum_ = sum_ + xin * yin R = sum_ / (n - 1) return R def find_sensor(self, tuple_): """used to find time traveled on a link, this finds the sensor_name in group sensor data""" if type(tuple_[1]) == tuple: sensors = [] for tuples in tuple_: sensors.append(tuples[1]) return sorted(sensors)[0] return tuple_[1] def split_sensors(self, list_): """splits the time data in a group of 50 into sensors instead of trips""" sensors = {} keys = [] for signal in list_[0][1]: sensors[self.find_sensor(signal)] = [] keys.append(self.find_sensor(signal)) for trip in list_: i = 0 for signal in trip[1]: long_time = self.get_time_value(signal) sensors[keys[i]].append(long_time) i = i + 1 return sensors, keys def time_between(self, sensor_list): """finds the time between two sensors""" time_values = [] sensors, keys = sensor_list for i in xrange(len(keys) - 1): times = [] link = keys[i] + '_' + keys[i + 1] sensor1 = sensors[keys[i]] sensor2 = sensors[keys[i + 1]] for i in xrange(len(sensor1)): time = self.diff_time(sensor1[i], sensor2[i]) times.append(time) time_values.append((link, times)) return time_values def get_trip_times(self, time_list): trips = {} for i in range(len(time_list[0])): times = [link[i] for link in time_list] trips[i + 1] = times return trips def find_totals(self, sensor_list): all_totals = [] #print sensor_list for j in range(len(sensor_list[0][1])): total = 0 for i in range(len(sensor_list)): # print sensor_list[i][j] total += sensor_list[i][1][j] all_totals.append(total) return all_totals def filter_data(self, sensors, group): sensor_data = [sensor_dt[0] for sensor_dt in sensors] times = [sensor_dt[1] for sensor_dt in sensors] info = self.get_trip_times(times) # print "Info", info fi = Filter.Final(info) trips_new, self.outliers[group] = fi.get_new_trips() #print "New", trips_new #print len(trips_new) print "removed: ", len(self.outliers[group]) temp = [] for sensor in sensor_data: temp.append((sensor, [])) #print temp for trip in sorted(trips_new.keys()): for i in range(len(trips_new[trip])): temp[i][1].append(trips_new[trip][i]) return temp def get_excel_data(self): new_data = {} sensor_times = {} total_time = {} book = open_workbook("I-5_S_BlueStats 39-9-10-11.xls") sheet = book.sheet_by_name('Data-1') mac_ids = sheet.col_values(0, start_rowx=1) starttime = sheet.col_values(4, start_rowx=1) tt_39_09 = sheet.col_values(7, start_rowx=1) tt_09_10 = sheet.col_values(11, start_rowx=1) tt_10_11 = sheet.col_values(15, start_rowx=1) totals = sheet.col_values(17, start_rowx=1) go = True last = False start = 0 end = 50 g_index = 0 while go: if last: go = False g_index += 1 group = "Group " + str(g_index) stimes = starttime[start:end] t1 = tt_39_09[start:end] t2 = tt_09_10[start:end] t3 = tt_10_11[start:end] total = totals[start:end] sensors = [('39_09', t1), ('09_10', t2), ('10_11', t3)] sorted_sensors = self.filter_data(sensors, group) Rvalues = [] j = 0 while j < len(sorted_sensors) - 1: (link1, times1) = sorted_sensors[j] (link2, times2) = sorted_sensors[j + 1] link = 'tt_' + link1 + '/' + link2 #print link R = self.find_R(times1, times2) Rvalues.append((link, R)) j = j + 1 new_data[group] = [r[1] for r in Rvalues] sensor_times[group] = sorted_sensors total_time[group] = total start = end end += 50 if end >= len(mac_ids): end = len(mac_ids) last = True g_index += 1 return new_data, None, sensor_times, total_time def compile_data(self, dict_, type_=None, filter_=True): """compiles all data ("R" value, Timespan of group (first starttime: laststarttime)), the average time and stdv for each link, the average trip time, trip stdv) and returns in dictionary. Also returns sensor times and total times""" data = {} sensor_times = {} total_time = {} for group in dict_: Rvalues = [] j = 0 sensorlist = self.split_sensors(dict_[group]) sensors = self.time_between(sensorlist) #print "sensors" , sensors if filter_: sensors = self.filter_data(sensors, group) sensor_times[group] = sensors totals = self.find_totals(sensors) total_time[group] = totals while j < len(sensors) - 1: (link1, times1) = sensors[j] (link2, times2) = sensors[j + 1] link = 'tt_' + link1 + '/' + link2 #print link R = self.find_R(times1, times2) Rvalues.append((link, R)) j = j + 1 start, end = self.list_times(dict_[group], type_) avgtimes = [] for sensor in sensors: avg, stdv = self.get_avg_stdv(sensor[1]) avgtimes.append((sensor[0], avg, stdv)) timespan = self.timespan(start) average, stdv = self.get_avg_stdv(totals) #, 'Start,End,Total(min)':times, data[group] = { 'R': Rvalues, 'Timespan': timespan, 'Segment:(Average time(sec), Average stdv)': avgtimes, 'Average_trip(min)': average, 'Stdv(min)': stdv } print "points removed:", self.count_items(self.outliers) return data, sensor_times, total_time def get_group_rvalues(self, sensors, type_='time', reverse=False): long_, longtime, l_times = self.get_long(sensors, type_=type_, reverse=reverse) GR = {} for group in long_: rlist = long_[group]['R'] rs = [r[1] for r in rlist] #print rs GR[group] = rs return GR, long_, longtime, l_times def cluster_data(self, sensors, type_='time', reverse=False): GR, long_, longtime, l_times = self.get_group_rvalues(sensors, type_=None, reverse=reverse) #GR, long_, longtime, l_times = self.get_excel_data() self.c_object = Cluster(GR) gc = self.c_object.main() total_times = {} link_times = {} for cluster in gc: link_times[cluster] = {} total_times[cluster] = [] for i in range(len(longtime[longtime.keys()[0]])): link_times[cluster][i + 1] = [] for group in gc[cluster]: for i in range(len(longtime[group])): ti = longtime[group][i][1] link_times[cluster][i + 1].extend(ti) total_times[cluster].extend(l_times[group]) print "trips in cluster " + str(cluster), len(total_times[cluster]) #print "c_object_1", c_object return gc, link_times, total_times def load_segment(self, sensors, type0_='time'): """loads a segment of grid""" self.load_data('SortedTrips.csv') self.get_trips(sensors) self.sort_trips(type_=type0_) self.split_trips(20) foward, forward_time, tottimes_f = self.compile_data( self.forward_split, type_=type0_) backward, backward_time, tottimes_b = self.compile_data( self.backward_split, type_=type0_) return foward, backward, forward_time, backward_time, tottimes_f, tottimes_b def get_long(self, sensors, type_=None, reverse=False): """get the direction of segment with the most trips""" forward, backward, forward_time, backward_time, tottimes_f, tottimes_b = self.load_segment( sensors, type0_=type_) if len(forward) >= len(backward): long_ = forward longtime = forward_time l_times = tottimes_f short = backward shorttime = backward_time s_times = tottimes_b else: long_ = backward longtime = backward_time l_times = tottimes_b short = forward shorttime = forward_time s_times = tottimes_f if reverse: long_ = short longtime = shorttime l_times = s_times return long_, longtime, l_times """finds begining and end of list of times""" def timespan(self, times): s_times = sorted(times) return (s_times[0].isoformat(), s_times[-1].isoformat()) def save_file(self, filename, dict_, del_): with open(filename + ".csv", 'wb') as f: writer = csv.writer(f) writer = csv.writer(f, delimiter=del_) for row in dict_.iteritems(): writer.writerow(row) def save_R_values(self, dicts): with open('Rvalues.csv', 'wb') as f: for direction in dicts: for group in direction.keys(): writer = csv.writer(f) writer.writerow('') writer.writerow(direction[group]['Start,End,Total(min)']) fieldnames = direction[group]['R'].keys() writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() writer.writerow(direction[group]['R']) def main(self, streets, type_='time'): """returns data from a dictionary containing streets: {segmentname:[sensors...], ....}""" for street in streets: da = DataAnalysis(pathname) forward, backward, foward_time, backward_time, tottimes_f, tottimes_b = da.load_segment( streets[street], type0_=type_) if len(forward) >= len(backward): longest = forward else: longest = backward print street, len(longest) da.save_file(street, longest, ';')
from datetime import datetime # Parse command line arguments args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running", "--keep-logs", "--alternate-version-labels-file"}) Utils.Debug=args.v killAll=args.clean_run dumpErrorDetails=args.dump_error_details dontKill=args.leave_running killLtnInstances=not dontKill killWallet=not dontKill keepLogs=args.keep_logs alternateVersionLabelsFile=args.alternate_version_labels_file walletMgr=WalletMgr(True) cluster=Cluster(walletd=True) cluster.setWalletMgr(walletMgr) def restartNode(node: Node, nodeId, chainArg=None, addOrSwapFlags=None, nodltnPath=None): if not node.killed: node.kill(signal.SIGTERM) isRelaunchSuccess = node.relaunch(nodeId, chainArg, addOrSwapFlags=addOrSwapFlags, timeout=5, cachePopen=True, nodltnPath=nodltnPath) assert isRelaunchSuccess, "Fail to relaunch" def shouldNodeContainPreactivateFeature(node): preactivateFeatureDigest = node.getSupportedProtocolFeatureDict()["PREACTIVATE_FEATURE"]["feature_digest"] assert preactivateFeatureDigest blockHeaderState = node.getLatestBlockHeaderState() activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"] return preactivateFeatureDigest in activatedProtocolFeatures
def create_system(options, full_system, system, dma_devices, ruby_system): if not buildEnv['GPGPU_SIM']: m5.util.panic("This script requires GPGPU-Sim integration to be built.") # Run the protocol script to setup CPU cluster, directory and DMA (all_sequencers, dir_cntrls, dma_cntrls, cpu_cluster) = \ VI_hammer.create_system(options, full_system, system, dma_devices, ruby_system) cpu_cntrl_count = len(cpu_cluster) + len(dir_cntrls) # # Build GPU cluster # # Empirically, Fermi per-core bandwidth peaks at roughly 23GB/s # (32B/cycle @ 772MHz). Use ~16B per Ruby cycle to match this. Maxwell # per-core bandwidth peaks at 40GB/s (42B/cycle @ 1029MHz). Use ~24B per # Ruby cycle to match this. if options.gpu_core_config == 'Fermi': l1_cluster_bw = 16 elif options.gpu_core_config == 'Maxwell': l1_cluster_bw = 24 elif options.gpu_core_config == 'Tegra': #FIXME using Fermi for now l1_cluster_bw = 16 else: m5.util.fatal("Unknown GPU core config: %s" % options.gpu_core_config) gpu_cluster = Cluster(intBW = l1_cluster_bw, extBW = l1_cluster_bw) gpu_cluster.disableConnectToParent() l2_bits = int(math.log(options.gpu_num_l2caches, 2)) block_size_bits = int(math.log(options.cacheline_size, 2)) # This represents the L1 to L2 interconnect latency # NOTES! 1) This latency is in Ruby (cache) cycles, not SM cycles # 2) Since the cluster interconnect doesn't model multihop latencies, # model these latencies with the controller latency variables. If # the interconnect model is changed, latencies will need to be # adjusted for reasonable total memory access delay. per_hop_interconnect_latency = 45 # ~15 GPU cycles num_dance_hall_hops = int(math.log(options.num_sc, 2)) if num_dance_hall_hops == 0: num_dance_hall_hops = 1 l1_to_l2_noc_latency = per_hop_interconnect_latency * num_dance_hall_hops # # Caches for GPU cores # for i in xrange(options.num_sc): # # First create the Ruby objects associated with the GPU cores # data_cache = L1Cache(size = options.sc_l1_size, assoc = options.sc_l1_assoc, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = False) data_l1_cntrl = GPUL1Cache_Controller(version = i*2, cache = data_cache, l2_select_num_bits = l2_bits, num_l2 = options.gpu_num_l2caches, transitions_per_cycle = options.ports, issue_latency = l1_to_l2_noc_latency, number_of_TBEs = options.gpu_l1_buf_depth, ruby_system = ruby_system) data_gpu_seq = RubySequencer(version = options.num_cpus + i*2, icache = data_cache, dcache = data_cache, max_outstanding_requests = options.gpu_l1_buf_depth, ruby_system = ruby_system, deadlock_threshold = 2000000, connect_to_io = False) tex_cache = L1Cache(size = options.sc_tl1_size, assoc = options.sc_tl1_assoc, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = False) tex_l1_cntrl = GPUL1Cache_Controller(version = i*2+1, cache = tex_cache, l2_select_num_bits = l2_bits, num_l2 = options.gpu_num_l2caches, transitions_per_cycle = options.ports, issue_latency = l1_to_l2_noc_latency, number_of_TBEs = options.gpu_tl1_buf_depth, ruby_system = ruby_system) tex_gpu_seq = RubySequencer(version = options.num_cpus + i*2+1, icache = tex_cache, dcache = tex_cache, max_outstanding_requests = options.gpu_tl1_buf_depth, ruby_system = ruby_system, deadlock_threshold = 2000000, connect_to_io = False) data_l1_cntrl.sequencer = data_gpu_seq tex_l1_cntrl.sequencer = tex_gpu_seq data_i = i*2; tex_i = i*2 +1; exec("ruby_system.l1_cntrl_sp%02d = data_l1_cntrl" % data_i) exec("ruby_system.l1_cntrl_sp%02d = tex_l1_cntrl" % tex_i) # # Add controllers and sequencers to the appropriate lists # all_sequencers.append(data_gpu_seq) all_sequencers.append(tex_gpu_seq) gpu_cluster.add(data_l1_cntrl) gpu_cluster.add(tex_l1_cntrl) # Connect the controllers to the network data_l1_cntrl.requestFromL1Cache = MessageBuffer(ordered = True) data_l1_cntrl.requestFromL1Cache.master = ruby_system.network.slave data_l1_cntrl.responseToL1Cache = MessageBuffer(ordered = True) data_l1_cntrl.responseToL1Cache.slave = ruby_system.network.master data_l1_cntrl.mandatoryQueue = MessageBuffer() tex_l1_cntrl.requestFromL1Cache = MessageBuffer(ordered = True) tex_l1_cntrl.requestFromL1Cache.master = ruby_system.network.slave tex_l1_cntrl.responseToL1Cache = MessageBuffer(ordered = True) tex_l1_cntrl.responseToL1Cache.slave = ruby_system.network.master tex_l1_cntrl.mandatoryQueue = MessageBuffer() l2_index_start = block_size_bits + l2_bits # Use L2 cache and interconnect latencies to calculate protocol latencies # NOTES! 1) These latencies are in Ruby (cache) cycles, not SM cycles # 2) Since the cluster interconnect doesn't model multihop latencies, # model these latencies with the controller latency variables. If # the interconnect model is changed, latencies will need to be # adjusted for reasonable total memory access delay. l2_cache_access_latency = 30 # ~10 GPU cycles l2_to_l1_noc_latency = per_hop_interconnect_latency * num_dance_hall_hops l2_to_mem_noc_latency = 125 # ~40 GPU cycles # Empirically, Fermi per-L2 bank bandwidth peaks at roughly 66GB/s # (92B/cycle @ 772MHz). Use ~34B per Ruby cycle to match this. Maxwell # per-L2 bank bandwidth peaks at 123GB/s (128B/cycle @ 1029MHz). Use ~64B # per Ruby cycle to match this. if options.gpu_core_config == 'Fermi': l2_cluster_bw = 34 elif options.gpu_core_config == 'Maxwell': l2_cluster_bw = 68 elif options.gpu_core_config == 'Tegra': #FIXME using Fermi configs for now l2_cluster_bw = 34 else: m5.util.fatal("Unknown GPU core config: %s" % options.gpu_core_config) l2_clusters = [] for i in xrange(options.gpu_num_l2caches): # # First create the Ruby objects associated with this cpu # l2_cache = L2Cache(size = options.sc_l2_size, assoc = options.sc_l2_assoc, start_index_bit = l2_index_start, replacement_policy = LRUReplacementPolicy(), dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = options.gpu_l2_resource_stalls) l2_cntrl = GPUL2Cache_Controller(version = i, L2cache = l2_cache, transitions_per_cycle = options.ports, l2_response_latency = l2_cache_access_latency + l2_to_l1_noc_latency, l2_request_latency = l2_to_mem_noc_latency, cache_response_latency = l2_cache_access_latency, ruby_system = ruby_system) exec("ruby_system.l2_cntrl%d = l2_cntrl" % i) l2_cluster = Cluster(intBW = l2_cluster_bw, extBW = l2_cluster_bw) l2_cluster.add(l2_cntrl) gpu_cluster.add(l2_cluster) l2_clusters.append(l2_cluster) # Connect the controller to the network l2_cntrl.responseToL1Cache = MessageBuffer(ordered = True) l2_cntrl.responseToL1Cache.master = ruby_system.network.slave l2_cntrl.requestFromCache = MessageBuffer() l2_cntrl.requestFromCache.master = ruby_system.network.slave l2_cntrl.responseFromCache = MessageBuffer() l2_cntrl.responseFromCache.master = ruby_system.network.slave l2_cntrl.unblockFromCache = MessageBuffer() l2_cntrl.unblockFromCache.master = ruby_system.network.slave l2_cntrl.requestFromL1Cache = MessageBuffer(ordered = True) l2_cntrl.requestFromL1Cache.slave = ruby_system.network.master l2_cntrl.forwardToCache = MessageBuffer() l2_cntrl.forwardToCache.slave = ruby_system.network.master l2_cntrl.responseToCache = MessageBuffer() l2_cntrl.responseToCache.slave = ruby_system.network.master l2_cntrl.triggerQueue = MessageBuffer() ############################################################################ # Pagewalk cache # NOTE: We use a CPU L1 cache controller here. This is to facilatate MMU # cache coherence (as the GPU L1 caches are incoherent without flushes # The L2 cache is small, and should have minimal affect on the # performance (see Section 6.2 of Power et al. HPCA 2014). pwd_cache = L1Cache(size = options.pwc_size, assoc = options.pwc_assoc, replacement_policy = options.pwc_policy, start_index_bit = block_size_bits, resourceStalls = False) # Small cache since CPU L1 requires I and D pwi_cache = L1Cache(size = "512B", assoc = 2, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, resourceStalls = False) # Small cache since CPU L1 controller requires L2 l2_cache = L2Cache(size = "512B", assoc = 2, start_index_bit = block_size_bits, resourceStalls = False) l1_cntrl = L1Cache_Controller(version = options.num_cpus, L1Icache = pwi_cache, L1Dcache = pwd_cache, L2cache = l2_cache, send_evictions = False, transitions_per_cycle = options.ports, issue_latency = l1_to_l2_noc_latency, cache_response_latency = 1, l2_cache_hit_latency = 1, number_of_TBEs = options.gpu_l1_buf_depth, ruby_system = ruby_system) cpu_seq = RubySequencer(version = options.num_cpus + options.num_sc*2, icache = pwd_cache, # Never get data from pwi_cache dcache = pwd_cache, dcache_hit_latency = 8, icache_hit_latency = 8, max_outstanding_requests = options.gpu_l1_buf_depth, ruby_system = ruby_system, deadlock_threshold = 2000000, connect_to_io = False) l1_cntrl.sequencer = cpu_seq ruby_system.l1_pw_cntrl = l1_cntrl all_sequencers.append(cpu_seq) gpu_cluster.add(l1_cntrl) # Connect the L1 controller and the network # Connect the buffers from the controller to network l1_cntrl.requestFromCache = MessageBuffer() l1_cntrl.requestFromCache.master = ruby_system.network.slave l1_cntrl.responseFromCache = MessageBuffer() l1_cntrl.responseFromCache.master = ruby_system.network.slave l1_cntrl.unblockFromCache = MessageBuffer() l1_cntrl.unblockFromCache.master = ruby_system.network.slave # Connect the buffers from the network to the controller l1_cntrl.forwardToCache = MessageBuffer() l1_cntrl.forwardToCache.slave = ruby_system.network.master l1_cntrl.responseToCache = MessageBuffer() l1_cntrl.responseToCache.slave = ruby_system.network.master l1_cntrl.mandatoryQueue = MessageBuffer() l1_cntrl.triggerQueue = MessageBuffer() # # Create controller for the copy engine to connect to in GPU cluster # Cache is unused by controller # cache = L1Cache(size = "4096B", assoc = 2) # Setting options.ce_buffering = 0 indicates that the CE can use infinite # buffering, but we need to specify a finite number of outstandng accesses # that the CE is allowed to issue. Just set it to some large number greater # than normal memory access latencies to ensure that the sequencer could # service one access per cycle. max_out_reqs = options.ce_buffering if max_out_reqs == 0: max_out_reqs = 1024 gpu_ce_seq = RubySequencer(version = options.num_cpus + options.num_sc*2 +1, icache = cache, dcache = cache, max_outstanding_requests = max_out_reqs, support_inst_reqs = False, ruby_system = ruby_system, connect_to_io = False) gpu_ce_cntrl = GPUCopyDMA_Controller(version = 0, sequencer = gpu_ce_seq, transitions_per_cycle = options.ports, number_of_TBEs = max_out_reqs, ruby_system = ruby_system) gpu_ce_cntrl.responseFromDir = MessageBuffer(ordered = True) gpu_ce_cntrl.responseFromDir.slave = ruby_system.network.master gpu_ce_cntrl.reqToDirectory = MessageBuffer(ordered = True) gpu_ce_cntrl.reqToDirectory.master = ruby_system.network.slave gpu_ce_cntrl.mandatoryQueue = MessageBuffer() ruby_system.ce_cntrl = gpu_ce_cntrl all_sequencers.append(gpu_ce_seq) # To limit the copy engine's bandwidth, we add it to a limited bandwidth # cluster. Approximate settings are as follows (assuming 2GHz Ruby clock): # PCIe v1.x x16 effective bandwidth ~= 4GB/s: intBW = 3, extBW = 3 # PCIe v2.x x16 effective bandwidth ~= 8GB/s: intBW = 5, extBW = 5 # PCIe v3.x x16 effective bandwidth ~= 16GB/s: intBW = 10, extBW = 10 # PCIe v4.x x16 effective bandwidth ~= 32GB/s: intBW = 21, extBW = 21 # NOTE: Bandwidth may bottleneck at other parts of the memory hierarchy, # so bandwidth considerations should be made in other parts of the memory # hierarchy also. gpu_ce_cluster = Cluster(intBW = 10, extBW = 10) gpu_ce_cluster.add(gpu_ce_cntrl) #z cache z_cache = L1Cache(size = options.sc_zl1_size, assoc = options.sc_zl1_assoc, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, dataArrayBanks = 8, tagArrayBanks = 8, dataAccessLatency = 1, tagAccessLatency = 1, resourceStalls = False) z_cntrl = GPUL1Cache_Controller(version = options.num_sc*2, cache = z_cache, l2_select_num_bits = l2_bits, num_l2 = options.gpu_num_l2caches, issue_latency = l1_to_l2_noc_latency, number_of_TBEs = options.gpu_zl1_buf_depth, ruby_system = ruby_system) z_seq = RubySequencer(version = options.num_cpus + options.num_sc*2+2, icache = z_cache, dcache = z_cache, max_outstanding_requests = options.gpu_zl1_buf_depth, ruby_system = ruby_system, deadlock_threshold = 2000000, connect_to_io = False) z_cntrl.sequencer = z_seq ruby_system.l1z_cntrl = z_cntrl all_sequencers.append(z_seq) gpu_cluster.add(z_cntrl) z_cntrl.requestFromL1Cache = MessageBuffer(ordered = True) z_cntrl.requestFromL1Cache.master = ruby_system.network.slave z_cntrl.responseToL1Cache = MessageBuffer(ordered = True) z_cntrl.responseToL1Cache.slave = ruby_system.network.master z_cntrl.mandatoryQueue = MessageBuffer() #z cache acl_cntrls = [] if options.accel_cfg_file: for idx, datapath in enumerate(system.datapaths): acl_cache = L1Cache(size = str(datapath.cacheSize), assoc = datapath.cacheAssoc, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, dataAccessLatency = datapath.cacheHitLatency) acli_cache = L1Cache(size = "512B", assoc = 2, replacement_policy = LRUReplacementPolicy(), start_index_bit = block_size_bits, dataAccessLatency = datapath.cacheHitLatency) #l2 cache to satisfy ruby l2_cache = L2Cache(size = "512B", #size = str(datapath.cacheSize), assoc = 2, #assoc = datapath.cacheAssoc, start_index_bit = block_size_bits) assert (not options.is_perfect_cache) #TODO: handle this option acl_cntrl = L1Cache_Controller(version = options.num_cpus+idx+1, L1Dcache = acl_cache, L1Icache = acli_cache, #never used L2cache = l2_cache, no_mig_atomic = not options.allow_atomic_migration, send_evictions = send_evicts(options), transitions_per_cycle = options.ports, ruby_system = ruby_system) acl_seq = RubySequencer(version = options.num_cpus + options.num_sc*2+3+idx, icache = acl_cache, dcache = acl_cache, ruby_system = ruby_system, deadlock_threshold = 2000000) # Connect the L1 controller and the network # Connect the buffers from the controller to network acl_cntrl.requestFromCache = MessageBuffer() acl_cntrl.requestFromCache.master = ruby_system.network.slave acl_cntrl.responseFromCache = MessageBuffer() acl_cntrl.responseFromCache.master = ruby_system.network.slave acl_cntrl.unblockFromCache = MessageBuffer() acl_cntrl.unblockFromCache.master = ruby_system.network.slave # Connect the buffers from the network to the controller acl_cntrl.forwardToCache = MessageBuffer() acl_cntrl.forwardToCache.slave = ruby_system.network.master acl_cntrl.responseToCache = MessageBuffer() acl_cntrl.responseToCache.slave = ruby_system.network.master acl_cntrl.mandatoryQueue = MessageBuffer() acl_cntrl.triggerQueue = MessageBuffer() acl_cntrl.sequencer = acl_seq exec("ruby_system.acl_cntrl%02d = acl_cntrl" % idx) all_sequencers.append(acl_seq) acl_cntrls.append(acl_cntrl) complete_cluster = Cluster(intBW = 32, extBW = 32) complete_cluster.add(gpu_ce_cluster) complete_cluster.add(cpu_cluster) complete_cluster.add(gpu_cluster) for cntrl in dir_cntrls: complete_cluster.add(cntrl) for cntrl in dma_cntrls: complete_cluster.add(cntrl) for cntrl in acl_cntrls: complete_cluster.add(cntrl) for cluster in l2_clusters: complete_cluster.add(cluster) return (all_sequencers, dir_cntrls, complete_cluster)
dumpErrorDetails = args.dump_error_details onlyBios = args.only_bios killAll = args.clean_run Utils.Debug = debug killSeatInstances = not dontKill topo = "mesh" delay = 1 prodCount = 1 # producers per producer node pnodes = 1 total_nodes = pnodes actualTest = "tests/nodeseat_run_test.py" testSuccessful = False cluster = Cluster() try: Print("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() Print( "producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes - pnodes, topo, delay)) Print("Stand up cluster") if cluster.launch(pnodes, total_nodes, prodCount, topo, delay, onlyBios=onlyBios,
Print = Utils.Print errorExit = Utils.errorExit cmdError = Utils.cmdError args = TestHelper.parse_args( {"-v", "--clean-run", "--dump-error-details", "--keep-logs"}) debug = args.v killAll = args.clean_run killEosInstances = True keepLogs = args.keep_logs dumpErrorDetails = dumpErrorDetails = args.dump_error_details Utils.Debug = debug https_port = 5555 cluster = Cluster(walletd=True) testSuccessful = False ClientName = "cleos" timeout = .5 * 12 * 2 + 60 # time for finalization with 1 producer + 60 seconds padding Utils.setIrreversibleTimeout(timeout) try: TestHelper.printSystemInfo("BEGIN") Print("Stand up cluster") # standup cluster with HTTPS enabled, but not configured # HTTP should still work extraArgs = {0: "--https-server-address 127.0.0.1:5555"} # specificExtraNodeosArgs=extraArgs
import pandas as pd from Cluster import Cluster def normalize(data, min_tab, max_tab): for index, row in min_tab.iteritems(): data.at[index] = (data.at[index] - min_tab.at[index]) / ( max_tab.at[index] - min_tab.at[index]) return data input_file = "./zoo.data" df = pd.read_csv(input_file, header=None) dcat = df.describe().transpose() mins = dcat['min'] maxs = dcat['max'] clusters = [] for index, row in df.iterrows(): cluster = Cluster(data1=normalize(row, mins, maxs)) clusters.append(cluster)
nodesFile=args.nodes_file seed=args.seed dontKill=args.leave_running dumpErrorDetails=args.dump_error_details killAll=args.clean_run killWallet=not dontKill killEosInstances=not dontKill if nodesFile is not None: killEosInstances=False Utils.Debug=debug testSuccessful=False random.seed(seed) # Use a fixed seed for repeatability. cluster=Cluster(walletd=True) walletMgr=WalletMgr(True) try: cluster.setWalletMgr(walletMgr) if nodesFile is not None: jsonStr=None with open(nodesFile, "r") as f: jsonStr=f.read() if not cluster.initializeNodesFromJson(jsonStr): errorExit("Failed to initilize nodes from Json string.") total_nodes=len(cluster.getNodes()) else: cluster.killall(allInstances=killAll) cluster.cleanup()
return 0 if profit2 == 0 else float(profit1 / profit2) def toString(self): i = 0; s = '' for cluster in self.clusters: s += '\t' + str(i) + ' => ' + cluster.toString() + ',\n' i += 1; return '{\n%s}' % s if __name__ == "__main__": t1 = Transaction('a,b').getTransaction() t2 = Transaction('a,b,c').getTransaction() cluster1 = Cluster(t1) cluster1.addTransaction(t2) print 'cluster1:' print cluster1.getCluster() t3 = Transaction('e,f').getTransaction() t4 = Transaction('e,g,f').getTransaction() cluster2 = Cluster(t3) cluster2.addTransaction(t4) print 'cluster2:' print cluster2.getCluster() print "clusters:" clope = Clope() clusters = []
debug = args.v total_nodes = pnodes killCount = args.kill_count if args.kill_count > 0 else 1 killSignal = args.kill_sig killRsnInstances = not args.leave_running dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs killAll = args.clean_run p2pPlugin = args.p2p_plugin seed = 1 Utils.Debug = debug testSuccessful = False random.seed(seed) # Use a fixed seed for repeatability. cluster = Cluster(walletd=True) walletMgr = WalletMgr(True) try: TestHelper.printSystemInfo("BEGIN") cluster.setChainStrategy(chainSyncStrategyStr) cluster.setWalletMgr(walletMgr) cluster.killall(allInstances=killAll) cluster.cleanup() walletMgr.killall(allInstances=killAll) walletMgr.cleanup() Print( "producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s"
def status(role): cluster = Cluster() if len(role) > 0 : cluster = cluster.getServersByRole(role) click.echo(cluster.getReport())
def testGetNearestCluster(self): self.assertEquals( KMeans.get_nearest_cluster( [cluster, Cluster([Point(np.array([8, 8]))])], point), 0)
# Parse command line arguments args = TestHelper.parse_args({ "-v", "--clean-run", "--dump-error-details", "--leave-running", "--keep-logs" }) Utils.Debug = args.v killAll = args.clean_run dumpErrorDetails = args.dump_error_details dontKill = args.leave_running killrsnInstances = not dontKill killWallet = not dontKill keepLogs = args.keep_logs walletMgr = WalletMgr(True) cluster = Cluster(walletd=True) cluster.setWalletMgr(walletMgr) testSuccessful = False try: TestHelper.printSystemInfo("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() # The following is the list of chainbase objects that need to be verified: # - account_object (bootstrap) # - code_object (bootstrap) # - generated_transaction_object # - global_property_object # - key_value_object (bootstrap) # - protocol_state_object (bootstrap)
# The following test case will test the Protocol Feature JSON reader of the blockchain def restartNode(node: Node, nodeId, chainArg=None, addOrSwapFlags=None): if not node.killed: node.kill(signal.SIGTERM) isRelaunchSuccess = node.relaunch(nodeId, chainArg, addOrSwapFlags=addOrSwapFlags, timeout=5, cachePopen=True) assert isRelaunchSuccess, "Fail to relaunch" walletMgr = WalletMgr(True) cluster = Cluster(walletd=True) cluster.setWalletMgr(walletMgr) # List to contain the test result message testSuccessful = False try: TestHelper.printSystemInfo("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() cluster.launch(extraNodroxeArgs=" --plugin roxe::producer_api_plugin ", dontBootstrap=True, pfSetupPolicy=PFSetupPolicy.NONE) biosNode = cluster.biosNode # Modify the JSON file and then restart the node so it updates the internal state newSubjectiveRestrictions = {
killEosInstances=not args.leave_running dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs killAll=args.clean_run relaunchTimeout=10 # Don't want to set too big, trying to reduce test time, but needs to be large enough for test to finish before # restart re-creates this many blocks. numBlocksToProduceBeforeRelaunch=80 numBlocksToWaitBeforeChecking=20 Utils.Debug=debug testSuccessful=False seed=1 random.seed(seed) # Use a fixed seed for repeatability. cluster=Cluster(walletd=True) walletMgr=WalletMgr(True) cluster.setWalletMgr(walletMgr) def relaunchNode(node: Node, nodeId, chainArg="", skipGenesis=True, relaunchAssertMessage="Fail to relaunch"): isRelaunchSuccess=node.relaunch(nodeId, chainArg=chainArg, timeout=relaunchTimeout, skipGenesis=skipGenesis, cachePopen=True) time.sleep(1) # Give a second to replay or resync if needed assert isRelaunchSuccess, relaunchAssertMessage return isRelaunchSuccess try: TestHelper.printSystemInfo("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() walletMgr.killall(allInstances=killAll)
def create_system(options, full_system, system, dma_devices, ruby_system): if buildEnv['PROTOCOL'] != 'GPU_RfO': panic("This script requires the GPU_RfO protocol to be built.") cpu_sequencers = [] # # The ruby network creation expects the list of nodes in the system to be # consistent with the NetDest list. Therefore the l1 controller nodes # must be listed before the directory nodes and directory nodes before # dma nodes, etc. # cp_cntrl_nodes = [] tcp_cntrl_nodes = [] sqc_cntrl_nodes = [] tcc_cntrl_nodes = [] tccdir_cntrl_nodes = [] dir_cntrl_nodes = [] l3_cntrl_nodes = [] # # Must create the individual controllers before the network to ensure the # controller constructors are called before the network constructor # TCC_bits = int(math.log(options.num_tccs, 2)) # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu # Clusters mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s for i in xrange(options.num_dirs): dir_cntrl = DirCntrl(TCC_select_num_bits = TCC_bits) dir_cntrl.create(options, ruby_system, system) dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units #Enough TBEs for all TCP TBEs # Connect the Directory controller to the ruby network dir_cntrl.requestFromCores = MessageBuffer(ordered = True) dir_cntrl.requestFromCores.slave = ruby_system.network.master dir_cntrl.responseFromCores = MessageBuffer() dir_cntrl.responseFromCores.slave = ruby_system.network.master dir_cntrl.unblockFromCores = MessageBuffer() dir_cntrl.unblockFromCores.slave = ruby_system.network.master dir_cntrl.probeToCore = MessageBuffer() dir_cntrl.probeToCore.master = ruby_system.network.slave dir_cntrl.responseToCore = MessageBuffer() dir_cntrl.responseToCore.master = ruby_system.network.slave dir_cntrl.triggerQueue = MessageBuffer(ordered = True) dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True) dir_cntrl.responseFromMemory = MessageBuffer() exec("system.dir_cntrl%d = dir_cntrl" % i) dir_cntrl_nodes.append(dir_cntrl) mainCluster.add(dir_cntrl) # For an odd number of CPUs, still create the right number of controllers cpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s for i in xrange((options.num_cpus + 1) / 2): cp_cntrl = CPCntrl() cp_cntrl.create(options, ruby_system, system) exec("system.cp_cntrl%d = cp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1]) # Connect the CP controllers and the network cp_cntrl.requestFromCore = MessageBuffer() cp_cntrl.requestFromCore.master = ruby_system.network.slave cp_cntrl.responseFromCore = MessageBuffer() cp_cntrl.responseFromCore.master = ruby_system.network.slave cp_cntrl.unblockFromCore = MessageBuffer() cp_cntrl.unblockFromCore.master = ruby_system.network.slave cp_cntrl.probeToCore = MessageBuffer() cp_cntrl.probeToCore.slave = ruby_system.network.master cp_cntrl.responseToCore = MessageBuffer() cp_cntrl.responseToCore.slave = ruby_system.network.master cp_cntrl.mandatoryQueue = MessageBuffer() cp_cntrl.triggerQueue = MessageBuffer(ordered = True) cpuCluster.add(cp_cntrl) gpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s for i in xrange(options.num_compute_units): tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits, number_of_TBEs = 2560) # max outstanding requests tcp_cntrl.create(options, ruby_system, system) exec("system.tcp_cntrl%d = tcp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(tcp_cntrl.coalescer) tcp_cntrl_nodes.append(tcp_cntrl) # Connect the TCP controller to the ruby network tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcp_cntrl.requestFromTCP.master = ruby_system.network.slave tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True) tcp_cntrl.responseFromTCP.master = ruby_system.network.slave tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True) tcp_cntrl.unblockFromCore.master = ruby_system.network.slave tcp_cntrl.probeToTCP = MessageBuffer(ordered = True) tcp_cntrl.probeToTCP.slave = ruby_system.network.master tcp_cntrl.responseToTCP = MessageBuffer(ordered = True) tcp_cntrl.responseToTCP.slave = ruby_system.network.master tcp_cntrl.mandatoryQueue = MessageBuffer() gpuCluster.add(tcp_cntrl) for i in xrange(options.num_sqc): sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits) sqc_cntrl.create(options, ruby_system, system) exec("system.sqc_cntrl%d = sqc_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(sqc_cntrl.sequencer) # Connect the SQC controller to the ruby network sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True) sqc_cntrl.requestFromSQC.master = ruby_system.network.slave sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True) sqc_cntrl.responseFromSQC.master = ruby_system.network.slave sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True) sqc_cntrl.unblockFromCore.master = ruby_system.network.slave sqc_cntrl.probeToSQC = MessageBuffer(ordered = True) sqc_cntrl.probeToSQC.slave = ruby_system.network.master sqc_cntrl.responseToSQC = MessageBuffer(ordered = True) sqc_cntrl.responseToSQC.slave = ruby_system.network.master sqc_cntrl.mandatoryQueue = MessageBuffer() # SQC also in GPU cluster gpuCluster.add(sqc_cntrl) for i in xrange(options.num_cp): tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits, number_of_TBEs = 2560) # max outstanding requests tcp_cntrl.createCP(options, ruby_system, system) exec("system.tcp_cntrl%d = tcp_cntrl" % (options.num_compute_units + i)) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(tcp_cntrl.sequencer) tcp_cntrl_nodes.append(tcp_cntrl) # Connect the TCP controller to the ruby network tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcp_cntrl.requestFromTCP.master = ruby_system.network.slave tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True) tcp_cntrl.responseFromTCP.master = ruby_system.network.slave tcp_cntrl.unblockFromCore = MessageBuffer(ordered = True) tcp_cntrl.unblockFromCore.master = ruby_system.network.slave tcp_cntrl.probeToTCP = MessageBuffer(ordered = True) tcp_cntrl.probeToTCP.slave = ruby_system.network.master tcp_cntrl.responseToTCP = MessageBuffer(ordered = True) tcp_cntrl.responseToTCP.slave = ruby_system.network.master tcp_cntrl.mandatoryQueue = MessageBuffer() gpuCluster.add(tcp_cntrl) sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits) sqc_cntrl.createCP(options, ruby_system, system) exec("system.sqc_cntrl%d = sqc_cntrl" % (options.num_compute_units + i)) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(sqc_cntrl.sequencer) # Connect the SQC controller to the ruby network sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True) sqc_cntrl.requestFromSQC.master = ruby_system.network.slave sqc_cntrl.responseFromSQC = MessageBuffer(ordered = True) sqc_cntrl.responseFromSQC.master = ruby_system.network.slave sqc_cntrl.unblockFromCore = MessageBuffer(ordered = True) sqc_cntrl.unblockFromCore.master = ruby_system.network.slave sqc_cntrl.probeToSQC = MessageBuffer(ordered = True) sqc_cntrl.probeToSQC.slave = ruby_system.network.master sqc_cntrl.responseToSQC = MessageBuffer(ordered = True) sqc_cntrl.responseToSQC.slave = ruby_system.network.master sqc_cntrl.mandatoryQueue = MessageBuffer() # SQC also in GPU cluster gpuCluster.add(sqc_cntrl) for i in xrange(options.num_tccs): tcc_cntrl = TCCCntrl(TCC_select_num_bits = TCC_bits, number_of_TBEs = options.num_compute_units * 2560) #Enough TBEs for all TCP TBEs tcc_cntrl.create(options, ruby_system, system) tcc_cntrl_nodes.append(tcc_cntrl) tccdir_cntrl = TCCDirCntrl(TCC_select_num_bits = TCC_bits, number_of_TBEs = options.num_compute_units * 2560) #Enough TBEs for all TCP TBEs tccdir_cntrl.create(options, ruby_system, system) tccdir_cntrl_nodes.append(tccdir_cntrl) exec("system.tcc_cntrl%d = tcc_cntrl" % i) exec("system.tccdir_cntrl%d = tccdir_cntrl" % i) # connect all of the wire buffers between L3 and dirs up req_to_tccdir = RubyWireBuffer() resp_to_tccdir = RubyWireBuffer() tcc_unblock_to_tccdir = RubyWireBuffer() req_to_tcc = RubyWireBuffer() probe_to_tcc = RubyWireBuffer() resp_to_tcc = RubyWireBuffer() tcc_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir, tcc_unblock_to_tccdir, req_to_tcc, probe_to_tcc, resp_to_tcc) tccdir_cntrl.connectWireBuffers(req_to_tccdir, resp_to_tccdir, tcc_unblock_to_tccdir, req_to_tcc, probe_to_tcc, resp_to_tcc) # Connect the TCC controller to the ruby network tcc_cntrl.responseFromTCC = MessageBuffer(ordered = True) tcc_cntrl.responseFromTCC.master = ruby_system.network.slave tcc_cntrl.responseToTCC = MessageBuffer(ordered = True) tcc_cntrl.responseToTCC.slave = ruby_system.network.master # Connect the TCC Dir controller to the ruby network tccdir_cntrl.requestFromTCP = MessageBuffer(ordered = True) tccdir_cntrl.requestFromTCP.slave = ruby_system.network.master tccdir_cntrl.responseFromTCP = MessageBuffer(ordered = True) tccdir_cntrl.responseFromTCP.slave = ruby_system.network.master tccdir_cntrl.unblockFromTCP = MessageBuffer(ordered = True) tccdir_cntrl.unblockFromTCP.slave = ruby_system.network.master tccdir_cntrl.probeToCore = MessageBuffer(ordered = True) tccdir_cntrl.probeToCore.master = ruby_system.network.slave tccdir_cntrl.responseToCore = MessageBuffer(ordered = True) tccdir_cntrl.responseToCore.master = ruby_system.network.slave tccdir_cntrl.probeFromNB = MessageBuffer() tccdir_cntrl.probeFromNB.slave = ruby_system.network.master tccdir_cntrl.responseFromNB = MessageBuffer() tccdir_cntrl.responseFromNB.slave = ruby_system.network.master tccdir_cntrl.requestToNB = MessageBuffer() tccdir_cntrl.requestToNB.master = ruby_system.network.slave tccdir_cntrl.responseToNB = MessageBuffer() tccdir_cntrl.responseToNB.master = ruby_system.network.slave tccdir_cntrl.unblockToNB = MessageBuffer() tccdir_cntrl.unblockToNB.master = ruby_system.network.slave tccdir_cntrl.triggerQueue = MessageBuffer(ordered = True) # TCC cntrls added to the GPU cluster gpuCluster.add(tcc_cntrl) gpuCluster.add(tccdir_cntrl) # Assuming no DMA devices assert(len(dma_devices) == 0) # Add cpu/gpu clusters to main cluster mainCluster.add(cpuCluster) mainCluster.add(gpuCluster) ruby_system.network.number_of_virtual_networks = 10 return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
# The following test case will test the Protocol Feature JSON reader of the blockchain def restartNode(node: Node, nodeId, chainArg=None, addSwapFlags=None): if not node.killed: node.kill(signal.SIGTERM) isRelaunchSuccess = node.relaunch(nodeId, chainArg, addSwapFlags=addSwapFlags, timeout=5, cachePopen=True) assert isRelaunchSuccess, "Fail to relaunch" walletMgr = WalletMgr(True) cluster = Cluster(walletd=True) cluster.setWalletMgr(walletMgr) testSuccessful = False try: TestHelper.printSystemInfo("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() cluster.launch( extraNodeosArgs= " --plugin eosio::producer_api_plugin --http-max-response-time-ms 990000 ", dontBootstrap=True, pfSetupPolicy=PFSetupPolicy.NONE) biosNode = cluster.biosNode # Modify the JSON file and then restart the node so it updates the internal state
def create_system(options, full_system, system, dma_devices, ruby_system): if not buildEnv['GPGPU_SIM']: m5.util.panic("This script requires GPGPU-Sim integration to be built.") # Run the protocol script to setup CPU cluster, directory and DMA (all_sequencers, dir_cntrls, dma_cntrls, cpu_cluster) = \ VI_hammer.create_system(options, full_system, system, dma_devices, ruby_system) # If we're going to split the directories/memory controllers if options.num_dev_dirs > 0: cpu_cntrl_count = len(cpu_cluster) else: cpu_cntrl_count = len(cpu_cluster) + len(dir_cntrls) # # Create controller for the copy engine to connect to in CPU cluster # Cache is unused by controller # cache = L1Cache(size = "4096B", assoc = 2) cpu_ce_seq = RubySequencer(version = options.num_cpus + options.num_sc, icache = cache, dcache = cache, max_outstanding_requests = 64, ruby_system = ruby_system, connect_to_io = False) cpu_ce_cntrl = GPUCopyDMA_Controller(version = 0, sequencer = cpu_ce_seq, number_of_TBEs = 256, ruby_system = ruby_system) cpu_cntrl_count += 1 cpu_ce_cntrl.responseFromDir = ruby_system.network.master cpu_ce_cntrl.reqToDirectory = ruby_system.network.slave # # Build GPU cluster # gpu_cluster = Cluster(intBW = 32, extBW = 32) gpu_cluster.disableConnectToParent() l2_bits = int(math.log(options.num_l2caches, 2)) block_size_bits = int(math.log(options.cacheline_size, 2)) # This represents the L1 to L2 interconnect latency # NOTE! This latency is in Ruby (cache) cycles, not SM cycles per_hop_interconnect_latency = 45 # ~15 GPU cycles num_dance_hall_hops = int(math.log(options.num_sc, 2)) if num_dance_hall_hops == 0: num_dance_hall_hops = 1 l1_to_l2_noc_latency = per_hop_interconnect_latency * num_dance_hall_hops # # Caches for GPU cores # for i in xrange(options.num_sc): # # First create the Ruby objects associated with the GPU cores # cache = L1Cache(size = options.sc_l1_size, assoc = options.sc_l1_assoc, replacement_policy = "LRU", start_index_bit = block_size_bits, dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = False) l1_cntrl = GPUL1Cache_Controller(version = i, cache = cache, l2_select_num_bits = l2_bits, num_l2 = options.num_l2caches, issue_latency = l1_to_l2_noc_latency, number_of_TBEs = options.gpu_l1_buf_depth, ruby_system = ruby_system) gpu_seq = RubySequencer(version = options.num_cpus + i, icache = cache, dcache = cache, max_outstanding_requests = options.gpu_l1_buf_depth, ruby_system = ruby_system, deadlock_threshold = 2000000, connect_to_io = False) l1_cntrl.sequencer = gpu_seq exec("ruby_system.l1_cntrl_sp%02d = l1_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # all_sequencers.append(gpu_seq) gpu_cluster.add(l1_cntrl) # Connect the controller to the network l1_cntrl.requestFromL1Cache = ruby_system.network.slave l1_cntrl.responseToL1Cache = ruby_system.network.master l2_index_start = block_size_bits + l2_bits # Use L2 cache and interconnect latencies to calculate protocol latencies # NOTE! These latencies are in Ruby (cache) cycles, not SM cycles l2_cache_access_latency = 30 # ~10 GPU cycles l2_to_l1_noc_latency = per_hop_interconnect_latency * num_dance_hall_hops l2_to_mem_noc_latency = 125 # ~40 GPU cycles l2_clusters = [] for i in xrange(options.num_l2caches): # # First create the Ruby objects associated with this cpu # l2_cache = L2Cache(size = options.sc_l2_size, assoc = options.sc_l2_assoc, start_index_bit = l2_index_start, replacement_policy = "LRU", dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = options.gpu_l2_resource_stalls) region_buffer = regionBuffer_Obj(size = "8MB", assoc = 2^16, start_index_bit = l2_index_start, replacement_policy = "LRU", dataArrayBanks = 4, tagArrayBanks = 4, dataAccessLatency = 4, tagAccessLatency = 4, resourceStalls = options.gpu_l2_resource_stalls, regionSize = options.region_size) l2_cntrl = GPUL2Cache_Controller(version = i, L2cache = l2_cache, regionBuffer = region_buffer, l2_response_latency = l2_cache_access_latency + l2_to_l1_noc_latency, l2_request_latency = l2_to_mem_noc_latency, cache_response_latency = l2_cache_access_latency, ruby_system = ruby_system) exec("ruby_system.l2_cntrl%d = l2_cntrl" % i) l2_cluster = Cluster(intBW = 32, extBW = 32) l2_cluster.add(l2_cntrl) gpu_cluster.add(l2_cluster) l2_clusters.append(l2_cluster) # Connect the controller to the network l2_cntrl.responseToL1Cache = ruby_system.network.slave l2_cntrl.requestFromCache = ruby_system.network.slave l2_cntrl.responseFromCache = ruby_system.network.slave l2_cntrl.unblockFromCache = ruby_system.network.slave l2_cntrl.requestFromL1Cache = ruby_system.network.master l2_cntrl.forwardToCache = ruby_system.network.master l2_cntrl.responseToCache = ruby_system.network.master gpu_phys_mem_size = system.gpu.gpu_memory_range.size() if options.num_dev_dirs > 0: mem_module_size = gpu_phys_mem_size / options.num_dev_dirs # # determine size and index bits for probe filter # By default, the probe filter size is configured to be twice the # size of the L2 cache. # pf_size = MemorySize(options.sc_l2_size) pf_size.value = pf_size.value * 2 dir_bits = int(math.log(options.num_dev_dirs, 2)) pf_bits = int(math.log(pf_size.value, 2)) if options.numa_high_bit: if options.pf_on or options.dir_on: # if numa high bit explicitly set, make sure it does not overlap # with the probe filter index assert(options.numa_high_bit - dir_bits > pf_bits) # set the probe filter start bit to just above the block offset pf_start_bit = block_size_bits else: if dir_bits > 0: pf_start_bit = dir_bits + block_size_bits - 1 else: pf_start_bit = block_size_bits dev_dir_cntrls = [] dev_mem_ctrls = [] num_cpu_dirs = len(dir_cntrls) for i in xrange(options.num_dev_dirs): # # Create the Ruby objects associated with the directory controller # dir_version = i + num_cpu_dirs dir_size = MemorySize('0B') dir_size.value = mem_module_size pf = ProbeFilter(size = pf_size, assoc = 4, start_index_bit = pf_start_bit) dev_dir_cntrl = Directory_Controller(version = dir_version, directory = \ RubyDirectoryMemory( \ version = dir_version, size = dir_size, numa_high_bit = \ options.numa_high_bit, device_directory = True), probeFilter = pf, probe_filter_enabled = options.pf_on, full_bit_dir_enabled = options.dir_on, ruby_system = ruby_system) if options.recycle_latency: dev_dir_cntrl.recycle_latency = options.recycle_latency exec("ruby_system.dev_dir_cntrl%d = dev_dir_cntrl" % i) dev_dir_cntrls.append(dev_dir_cntrl) # Connect the directory controller to the network dev_dir_cntrl.forwardFromDir = ruby_system.network.slave dev_dir_cntrl.responseFromDir = ruby_system.network.slave dev_dir_cntrl.dmaResponseFromDir = ruby_system.network.slave dev_dir_cntrl.unblockToDir = ruby_system.network.master dev_dir_cntrl.responseToDir = ruby_system.network.master dev_dir_cntrl.requestToDir = ruby_system.network.master dev_dir_cntrl.dmaRequestToDir = ruby_system.network.master dev_mem_ctrl = MemConfig.create_mem_ctrl( MemConfig.get(options.mem_type), system.gpu.gpu_memory_range, i, options.num_dev_dirs, int(math.log(options.num_dev_dirs, 2)), options.cacheline_size) dev_mem_ctrl.port = dev_dir_cntrl.memory dev_mem_ctrls.append(dev_mem_ctrl) system.dev_mem_ctrls = dev_mem_ctrls else: # Since there are no device directories, use CPU directories # Fix up the memory sizes of the CPU directories num_dirs = len(dir_cntrls) add_gpu_mem = gpu_phys_mem_size / num_dirs for cntrl in dir_cntrls: new_size = cntrl.directory.size.value + add_gpu_mem cntrl.directory.size.value = new_size # # Create controller for the copy engine to connect to in GPU cluster # Cache is unused by controller # cache = L1Cache(size = "4096B", assoc = 2) gpu_ce_seq = RubySequencer(version = options.num_cpus + options.num_sc + 1, icache = cache, dcache = cache, max_outstanding_requests = 64, support_inst_reqs = False, ruby_system = ruby_system, connect_to_io = False) gpu_ce_cntrl = GPUCopyDMA_Controller(version = 1, sequencer = gpu_ce_seq, number_of_TBEs = 256, ruby_system = ruby_system) ruby_system.l1_cntrl_ce = gpu_ce_cntrl all_sequencers.append(cpu_ce_seq) all_sequencers.append(gpu_ce_seq) gpu_ce_cntrl.responseFromDir = ruby_system.network.master gpu_ce_cntrl.reqToDirectory = ruby_system.network.slave complete_cluster = Cluster(intBW = 32, extBW = 32) complete_cluster.add(cpu_ce_cntrl) complete_cluster.add(gpu_ce_cntrl) complete_cluster.add(cpu_cluster) complete_cluster.add(gpu_cluster) for cntrl in dir_cntrls: complete_cluster.add(cntrl) for cntrl in dev_dir_cntrls: complete_cluster.add(cntrl) for cntrl in dma_cntrls: complete_cluster.add(cntrl) for cluster in l2_clusters: complete_cluster.add(cluster) return (all_sequencers, dir_cntrls, complete_cluster)
def create_system(options, full_system, system, dma_devices, ruby_system): if buildEnv['PROTOCOL'] != 'MOESI_AMD_Base': panic("This script requires the MOESI_AMD_Base protocol.") cpu_sequencers = [] # # The ruby network creation expects the list of nodes in the system to # be consistent with the NetDest list. Therefore the l1 controller # nodes must be listed before the directory nodes and directory nodes # before dma nodes, etc. # l1_cntrl_nodes = [] l3_cntrl_nodes = [] dir_cntrl_nodes = [] control_count = 0 # # Must create the individual controllers before the network to ensure # the controller constructors are called before the network constructor # # This is the base crossbar that connects the L3s, Dirs, and cpu # Cluster mainCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s for i in xrange(options.num_dirs): dir_cntrl = DirCntrl(TCC_select_num_bits = 0) dir_cntrl.create(options, ruby_system, system) # Connect the Directory controller to the ruby network dir_cntrl.requestFromCores = MessageBuffer(ordered = True) dir_cntrl.requestFromCores.slave = ruby_system.network.master dir_cntrl.responseFromCores = MessageBuffer() dir_cntrl.responseFromCores.slave = ruby_system.network.master dir_cntrl.unblockFromCores = MessageBuffer() dir_cntrl.unblockFromCores.slave = ruby_system.network.master dir_cntrl.probeToCore = MessageBuffer() dir_cntrl.probeToCore.master = ruby_system.network.slave dir_cntrl.responseToCore = MessageBuffer() dir_cntrl.responseToCore.master = ruby_system.network.slave dir_cntrl.triggerQueue = MessageBuffer(ordered = True) dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True) dir_cntrl.responseFromMemory = MessageBuffer() exec("system.dir_cntrl%d = dir_cntrl" % i) dir_cntrl_nodes.append(dir_cntrl) mainCluster.add(dir_cntrl) # Technically this config can support an odd number of cpus, but the top # level config files, such as the ruby_random_tester, will get confused if # the number of cpus does not equal the number of sequencers. Thus make # sure that an even number of cpus is specified. assert((options.num_cpus % 2) == 0) # For an odd number of CPUs, still create the right number of controllers cpuCluster = Cluster(extBW = 512, intBW = 512) # 1 TB/s for i in xrange((options.num_cpus + 1) / 2): cp_cntrl = CPCntrl() cp_cntrl.create(options, ruby_system, system) exec("system.cp_cntrl%d = cp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1]) # Connect the CP controllers and the network cp_cntrl.requestFromCore = MessageBuffer() cp_cntrl.requestFromCore.master = ruby_system.network.slave cp_cntrl.responseFromCore = MessageBuffer() cp_cntrl.responseFromCore.master = ruby_system.network.slave cp_cntrl.unblockFromCore = MessageBuffer() cp_cntrl.unblockFromCore.master = ruby_system.network.slave cp_cntrl.probeToCore = MessageBuffer() cp_cntrl.probeToCore.slave = ruby_system.network.master cp_cntrl.responseToCore = MessageBuffer() cp_cntrl.responseToCore.slave = ruby_system.network.master cp_cntrl.mandatoryQueue = MessageBuffer() cp_cntrl.triggerQueue = MessageBuffer(ordered = True) cpuCluster.add(cp_cntrl) # Assuming no DMA devices assert(len(dma_devices) == 0) # Add cpu/gpu clusters to main cluster mainCluster.add(cpuCluster) ruby_system.network.number_of_virtual_networks = 10 return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
Utils.cmdError("only saw %s producers of expected 21. At blockNum %s only the following producers were seen: %s" % (len(prodsSeenKeys), blockNum, ",".join(prodsSeenKeys))) Utils.errorExit("Failed because of missing block producers") Utils.Debug=temp Print=Utils.Print errorExit=Utils.errorExit from core_symbol import CORE_SYMBOL args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run", "--p2p-plugin","--wallet-port"}) Utils.Debug=args.v totalNodes=4 cluster=Cluster(walletd=True) dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running prodCount=args.prod_count killAll=args.clean_run p2pPlugin=args.p2p_plugin walletPort=args.wallet_port walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill WalletdName=Utils.EosWalletName ClientName="cleon"
ResourceManager._cluster_dict = {} try: exist_cluster = ResourceManager._db.sync_from_db() for cluster in exist_cluster: ResourceManager.create_cluster( cluster["cluster_name"], cluster["protected_layers_string"]) if cluster["node_list"] != []: ResourceManager.add_node(cluster["cluster_name"], cluster["node_list"]) for instance in cluster["instance_list"]: ResourceManager.add_instance(cluster["cluster_name"], instance) logging.info("ResourceManager--synco from DB finish") except Exception as e: print str(e) logging.error("ClusterManagwer--synco from DB fail") @staticmethod def sync_to_database(): cluster_list = ResourceManager._cluster_dict ResourceManager._db.sync_to_db(cluster_list) if __name__ == "__main__": cluster_name = 'test' ResourceManager.init() b = Cluster(cluster_name) ResourceManager._cluster_dict[cluster_name] = b print(ResourceManager._cluster_dict)
def doMonitor(interval): cluster = Cluster() while True: click.echo(cluster.getReport()) click.echo("\n\n") sleep(interval)
""" Solves the N-body problem using RungeKutta or Velocity-Verlet """ from Cluster import Cluster if __name__ == "__main__": test = Cluster(N=4, n_steps=5) test.initialize(radius=100) test.animate()
def kafka(restart, start, stop, force_stop, clean, brokers, wait_before_start, wait_before_kill): cluster = Cluster() if len(brokers) > 0 : brokerList = brokers.split(",") cluster = cluster.getServersByHostname(brokerList) if(restart or stop): logging.debug("Shutting down Kafka") cluster.shutdownKafka() if(force_stop): logging.debug("Waiting for "+str(wait_before_kill)+" seconds") sleep(wait_before_kill) logging.debug("Force Killing Kafka") cluster.killKafka() if(clean): logging.debug("Cleaning Kafka") cluster.cleanKafka() if(restart or start): logging.debug("Waiting for "+str(wait_before_start)+" seconds") sleep(wait_before_start) logging.debug("Starting Kafka") cluster.startKafka() click.echo(cluster.getReport())
defproduceraPrvtKey = args.defproducera_prvt_key defproducerbPrvtKey = args.defproducerb_prvt_key dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontLaunch = args.dont_launch dontKill = args.leave_running prodCount = args.prod_count onlyBios = args.only_bios killAll = args.clean_run sanityTest = args.sanity_test p2pPlugin = args.p2p_plugin Utils.Debug = debug localTest = True if server == TestHelper.LOCAL_HOST else False cluster = Cluster(walletd=True, enableMongo=enableMongo, defproduceraPrvtKey=defproduceraPrvtKey, defproducerbPrvtKey=defproducerbPrvtKey) walletMgr = WalletMgr(True) testSuccessful = False killSeatInstances = not dontKill killWallet = not dontKill dontBootstrap = sanityTest WalletdName = "keyseatd" ClientName = "cliseat" timeout = .5 * 12 * 2 + 60 # time for finalization with 1 producer + 60 seconds padding Utils.setIrreversibleTimeout(timeout) try: TestHelper.printSystemInfo("BEGIN") Print("SERVER: %s" % (server))
file.close() def load_data(filename): """ Returns the object stored in the file @param filename: the name of the file @return the object stored in the file """ file = open(filename, "r") data = pickle.load(file) file.close() return data """ Example use of Advise.py scrapper.query(QUERY='university', OBJECT_TYPE='page') university_pages = scrapper.get_pages(QUERY='university', OBJECT_TYPE='page') scrapper.dump("query_data") store_data(university_pages, "university_pages") clusters = Cluster.kmeans(data=university_pages, k=20) """ if __name__ == "__main__": scrapper = Scrapper() scrapper.load("query_data") university_pages = load_data("university_pages") clusters = Cluster.kmeans(data=university_pages) visualizer = Visualizer(data=university_pages, clusters=clusters)
help="How many catchup-nodes to launch", default=10) extraArgs = appArgs.add(flag="--txn-gen-nodes", type=int, help="How many transaction generator nodes", default=2) args = TestHelper.parse_args( { "--prod-count", "--dump-error-details", "--keep-logs", "-v", "--leave-running", "--clean-run", "-p", "--wallet-port" }, applicationSpecificArgs=appArgs) Utils.Debug = args.v pnodes = args.p if args.p > 0 else 1 startedNonProdNodes = args.txn_gen_nodes if args.txn_gen_nodes >= 2 else 2 cluster = Cluster(walletd=True) dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontKill = args.leave_running prodCount = args.prod_count if args.prod_count > 1 else 2 killAll = args.clean_run walletPort = args.wallet_port catchupCount = args.catchup_count if args.catchup_count > 0 else 1 totalNodes = startedNonProdNodes + pnodes + catchupCount walletMgr = WalletMgr(True, port=walletPort) testSuccessful = False killscsInstances = not dontKill killWallet = not dontKill WalletdName = Utils.scsWalletName
""" Solves the N-body problem using RungeKutta or Velocity-Verlet """ from Cluster import Cluster if __name__ == "__main__": test = Cluster(N=40, n_steps=100) test.initialize(radius=20) test.animate()
info1=prodNodes[1].getInfo(exitOnError=True) headBlockNum=min(int(info0["head_block_num"]),int(info1["head_block_num"])) libNum=min(int(info0["last_irreversible_block_num"]), int(info1["last_irreversible_block_num"])) return (headBlockNum, libNum) args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run", "--wallet-port"}) Utils.Debug=args.v totalProducerNodes=2 totalNonProducerNodes=1 totalNodes=totalProducerNodes+totalNonProducerNodes maxActiveProducers=3 totalProducers=maxActiveProducers cluster=Cluster(walletd=True) dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running killAll=args.clean_run walletPort=args.wallet_port walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killscsInstances=not dontKill killWallet=not dontKill WalletdName=Utils.scsWalletName ClientName="clscs" try:
dumpErrorDetails=args.dump_error_details onlyBios=args.only_bios killAll=args.clean_run Utils.Debug=debug killEosInstances=not dontKill topo="mesh" delay=1 prodCount=1 # producers per producer node pnodes=1 total_nodes=pnodes actualTest="tests/nodeos_run_test.py" testSuccessful=False cluster=Cluster() try: Print("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() Print ("producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes-pnodes, topo, delay)) Print("Stand up cluster") if cluster.launch(pnodes, total_nodes, prodCount, topo, delay, onlyBios=onlyBios, dontKill=dontKill) is False: errorExit("Failed to stand up eos cluster.") Print ("Wait for Cluster stabilization") # wait for cluster to start producing blocks if not cluster.waitOnClusterBlockNumSync(3): errorExit("Cluster never stabilized")
testSuccessful = False clusterMapJsonTemplate = """{ "keys": { "defproduceraPrivateKey": "%s", "defproducerbPrivateKey": "%s" }, "nodes": [ {"port": 8888, "host": "localhost"}, {"port": 8889, "host": "localhost"}, {"port": 8890, "host": "localhost"} ] } """ cluster = Cluster(walletd=True) (fd, nodesFile) = tempfile.mkstemp() try: TestHelper.printSystemInfo("BEGIN") cluster.killall(allInstances=killAll) cluster.cleanup() Print( "producing nodes: %s, non-producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d" % (pnodes, total_nodes - pnodes, topo, delay)) Print("Stand up cluster") if cluster.launch(pnodes=pnodes, totalNodes=total_nodes, prodCount=prodCount, topo=topo,
def create_system(options, full_system, system, dma_ports, ruby_system): if 'VI_hammer' not in buildEnv['PROTOCOL']: panic("This script requires the VI_hammer protocol to be built.") options.access_backing_store = True cpu_sequencers = [] topology = Cluster(intBW = 32, extBW = 32) # # Must create the individual controllers before the network to ensure the # controller constructors are called before the network constructor # l2_bits_float = math.log(options.num_l2caches, 2) l2_bits = int(l2_bits_float) if l2_bits_float > l2_bits: l2_bits += 1 block_size_bits = int(math.log(options.cacheline_size, 2)) for i in xrange(options.num_cpus): # # First create the Ruby objects associated with this cpu # l1i_cache = L1Cache(size = options.l1i_size, assoc = options.l1i_assoc, start_index_bit = block_size_bits, is_icache = True) l1d_cache = L1Cache(size = options.l1d_size, assoc = options.l1d_assoc, start_index_bit = block_size_bits) l2_cache = L2Cache(size = "2MB", assoc = options.l2_assoc, start_index_bit = block_size_bits) #region_buffer = regionBuffer_Obj(size = "4MB", # assoc = 32768, # start_index_bit = l2_index_start, # replacement_policy = "LRU", # dataArrayBanks = 4, # tagArrayBanks = 4, # dataAccessLatency = 4, # tagAccessLatency = 4, # resourceStalls = options.gpu_l2_resource_stalls, # regionSize = options.region_size) l1_cntrl = L1Cache_Controller(version = i, L1Icache = l1i_cache, L1Dcache = l1d_cache, L2cache = l2_cache, no_mig_atomic = not \ options.allow_atomic_migration, send_evictions = ( options.cpu_type == "detailed"), ruby_system = ruby_system) cpu_seq = RubySequencer(version = i, icache = l1i_cache, dcache = l1d_cache, ruby_system = ruby_system) l1_cntrl.sequencer = cpu_seq if options.recycle_latency: l1_cntrl.recycle_latency = options.recycle_latency exec("ruby_system.l1_cntrl%d = l1_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(cpu_seq) topology.add(l1_cntrl) # Connect the L1 controller and the network # Connect the buffers from the controller to network l1_cntrl.requestFromCache = ruby_system.network.slave l1_cntrl.responseFromCache = ruby_system.network.slave l1_cntrl.unblockFromCache = ruby_system.network.slave # Connect the buffers from the network to the controller l1_cntrl.forwardToCache = ruby_system.network.master l1_cntrl.responseToCache = ruby_system.network.master cpu_mem_range = AddrRange(options.total_mem_size) mem_module_size = cpu_mem_range.size() / options.num_dirs # # determine size and index bits for probe filter # By default, the probe filter size is configured to be twice the # size of the L2 cache. # pf_size = MemorySize(options.l2_size) pf_size.value = pf_size.value * 2 dir_bits = int(math.log(options.num_dirs, 2)) pf_bits = int(math.log(pf_size.value, 2)) if options.numa_high_bit: if options.pf_on or options.dir_on: # if numa high bit explicitly set, make sure it does not overlap # with the probe filter index assert(options.numa_high_bit - dir_bits > pf_bits) # set the probe filter start bit to just above the block offset pf_start_bit = block_size_bits else: if dir_bits > 0: pf_start_bit = dir_bits + block_size_bits - 1 else: pf_start_bit = block_size_bits dir_cntrl_nodes = [] for i in xrange(options.num_dirs): # # Create the Ruby objects associated with the directory controller # dir_size = MemorySize('0B') dir_size.value = mem_module_size pf = ProbeFilter(size = pf_size, assoc = 4, start_index_bit = pf_start_bit) dir_cntrl = Directory_Controller(version = i, directory = \ RubyDirectoryMemory( \ version = i, size = dir_size, numa_high_bit = \ options.numa_high_bit), probeFilter = pf, probe_filter_enabled = options.pf_on, full_bit_dir_enabled = options.dir_on, ruby_system = ruby_system) if options.recycle_latency: dir_cntrl.recycle_latency = options.recycle_latency exec("ruby_system.dir_cntrl%d = dir_cntrl" % i) dir_cntrl_nodes.append(dir_cntrl) # Connect the directory controller to the network dir_cntrl.forwardFromDir = ruby_system.network.slave dir_cntrl.responseFromDir = ruby_system.network.slave dir_cntrl.dmaResponseFromDir = ruby_system.network.slave dir_cntrl.unblockToDir = ruby_system.network.master dir_cntrl.responseToDir = ruby_system.network.master dir_cntrl.requestToDir = ruby_system.network.master dir_cntrl.dmaRequestToDir = ruby_system.network.master dma_cntrl_nodes = [] for i, dma_port in enumerate(dma_ports): # # Create the Ruby objects associated with the dma controller # dma_seq = DMASequencer(version = i, ruby_system = ruby_system) dma_cntrl = DMA_Controller(version = i, dma_sequencer = dma_seq, ruby_system = ruby_system) exec("ruby_system.dma_cntrl%d = dma_cntrl" % i) exec("ruby_system.dma_cntrl%d.dma_sequencer.slave = dma_port" % i) dma_cntrl_nodes.append(dma_cntrl) if options.recycle_latency: dma_cntrl.recycle_latency = options.recycle_latency # Connect the dma controller to the network dma_cntrl.responseFromDir = ruby_system.network.master dma_cntrl.reqToDirectory = ruby_system.network.slave # Create the io controller and the sequencer if full_system: io_seq = DMASequencer(version=len(dma_ports), ruby_system=ruby_system) ruby_system._io_port = io_seq io_controller = DMA_Controller(version = len(dma_ports), dma_sequencer = io_seq, ruby_system = ruby_system) ruby_system.io_controller = io_controller # Connect the dma controller to the network io_controller.responseFromDir = ruby_system.network.master io_controller.reqToDirectory = ruby_system.network.slave dma_cntrl_nodes.append(io_controller) return (cpu_sequencers, dir_cntrl_nodes, dma_cntrl_nodes, topology)
nodesFile = args.nodes_file seed = args.seed dontKill = args.leave_running dumpErrorDetails = args.dump_error_details killAll = args.clean_run killWallet = not dontKill killEnuInstances = not dontKill if nodesFile is not None: killEnuInstances = False Utils.Debug = debug testSuccessful = False random.seed(seed) # Use a fixed seed for repeatability. cluster = Cluster(enuwalletd=True) walletMgr = WalletMgr(True) try: cluster.setWalletMgr(walletMgr) if nodesFile is not None: jsonStr = None with open(nodesFile, "r") as f: jsonStr = f.read() if not cluster.initializeNodesFromJson(jsonStr): errorExit("Failed to initilize nodes from Json string.") total_nodes = len(cluster.getNodes()) else: cluster.killall(allInstances=killAll) cluster.cleanup()
return retStr ############################################################### # nodeos_voting_test # --dump-error-details <Upon error print etc/yosemite/node_*/config.ini and var/lib/node_*/stderr.log to stdout> # --keep-logs <Don't delete var/lib/node_* folders upon test completion> ############################################################### args = TestHelper.parse_args({ "--dump-error-details", "--keep-logs", "-v", "--leave-running", "--clean-run", "--wallet-port" }) Utils.Debug = args.v totalNodes = 4 cluster = Cluster(walletd=True) dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontKill = args.leave_running killAll = args.clean_run walletPort = args.wallet_port walletMgr = WalletMgr(True, port=walletPort) testSuccessful = False killEosInstances = not dontKill killWallet = not dontKill WalletdName = Utils.EosWalletName ClientName = "cleos" try:
(len(prodsSeenKeys), prodsSize, blockNum, ",".join(prodsSeenKeys))) Utils.errorExit("Failed because of missing block producers") Utils.Debug = temp Print = Utils.Print errorExit = Utils.errorExit args = TestHelper.parse_args({ "--prod-count", "--dump-error-details", "--keep-logs", "-v", "--leave-running", "--clean-run", "--wallet-port" }) Utils.Debug = args.v totalNodes = 3 cluster = Cluster(walletd=True) dumpErrorDetails = args.dump_error_details keepLogs = args.keep_logs dontKill = args.leave_running prodCount = args.prod_count killAll = args.clean_run walletPort = args.wallet_port walletMgr = WalletMgr(True, port=walletPort) testSuccessful = False killEosInstances = not dontKill killWallet = not dontKill WalletdName = Utils.EosWalletName ClientName = "clio"
def create_system(options, full_system, system, dma_devices, ruby_system): if buildEnv['PROTOCOL'] != 'GPU_VIPER_Region': panic("This script requires the GPU_VIPER_Region protocol to be built.") cpu_sequencers = [] # # The ruby network creation expects the list of nodes in the system to be # consistent with the NetDest list. Therefore the l1 controller nodes # must be listed before the directory nodes and directory nodes before # dma nodes, etc. # dir_cntrl_nodes = [] # For an odd number of CPUs, still create the right number of controllers TCC_bits = int(math.log(options.num_tccs, 2)) # # Must create the individual controllers before the network to ensure the # controller constructors are called before the network constructor # # For an odd number of CPUs, still create the right number of controllers crossbar_bw = 16 * options.num_compute_units #Assuming a 2GHz clock cpuCluster = Cluster(extBW = (crossbar_bw), intBW=crossbar_bw) for i in xrange((options.num_cpus + 1) / 2): cp_cntrl = CPCntrl() cp_cntrl.create(options, ruby_system, system) rb_cntrl = RBCntrl() rb_cntrl.create(options, ruby_system, system) rb_cntrl.number_of_TBEs = 256 rb_cntrl.isOnCPU = True cp_cntrl.regionBufferNum = rb_cntrl.version exec("system.cp_cntrl%d = cp_cntrl" % i) exec("system.rb_cntrl%d = rb_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.extend([cp_cntrl.sequencer, cp_cntrl.sequencer1]) # Connect the CP controllers and the network cp_cntrl.requestFromCore = MessageBuffer() cp_cntrl.requestFromCore.master = ruby_system.network.slave cp_cntrl.responseFromCore = MessageBuffer() cp_cntrl.responseFromCore.master = ruby_system.network.slave cp_cntrl.unblockFromCore = MessageBuffer() cp_cntrl.unblockFromCore.master = ruby_system.network.slave cp_cntrl.probeToCore = MessageBuffer() cp_cntrl.probeToCore.slave = ruby_system.network.master cp_cntrl.responseToCore = MessageBuffer() cp_cntrl.responseToCore.slave = ruby_system.network.master cp_cntrl.mandatoryQueue = MessageBuffer() cp_cntrl.triggerQueue = MessageBuffer(ordered = True) # Connect the RB controllers to the ruby network rb_cntrl.requestFromCore = MessageBuffer(ordered = True) rb_cntrl.requestFromCore.slave = ruby_system.network.master rb_cntrl.responseFromCore = MessageBuffer() rb_cntrl.responseFromCore.slave = ruby_system.network.master rb_cntrl.requestToNetwork = MessageBuffer() rb_cntrl.requestToNetwork.master = ruby_system.network.slave rb_cntrl.notifyFromRegionDir = MessageBuffer() rb_cntrl.notifyFromRegionDir.slave = ruby_system.network.master rb_cntrl.probeFromRegionDir = MessageBuffer() rb_cntrl.probeFromRegionDir.slave = ruby_system.network.master rb_cntrl.unblockFromDir = MessageBuffer() rb_cntrl.unblockFromDir.slave = ruby_system.network.master rb_cntrl.responseToRegDir = MessageBuffer() rb_cntrl.responseToRegDir.master = ruby_system.network.slave rb_cntrl.triggerQueue = MessageBuffer(ordered = True) cpuCluster.add(cp_cntrl) cpuCluster.add(rb_cntrl) gpuCluster = Cluster(extBW = (crossbar_bw), intBW = crossbar_bw) for i in xrange(options.num_compute_units): tcp_cntrl = TCPCntrl(TCC_select_num_bits = TCC_bits, issue_latency = 1, number_of_TBEs = 2560) # TBEs set to max outstanding requests tcp_cntrl.create(options, ruby_system, system) tcp_cntrl.WB = options.WB_L1 tcp_cntrl.disableL1 = False exec("system.tcp_cntrl%d = tcp_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(tcp_cntrl.coalescer) # Connect the CP (TCP) controllers to the ruby network tcp_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcp_cntrl.requestFromTCP.master = ruby_system.network.slave tcp_cntrl.responseFromTCP = MessageBuffer(ordered = True) tcp_cntrl.responseFromTCP.master = ruby_system.network.slave tcp_cntrl.unblockFromCore = MessageBuffer() tcp_cntrl.unblockFromCore.master = ruby_system.network.slave tcp_cntrl.probeToTCP = MessageBuffer(ordered = True) tcp_cntrl.probeToTCP.slave = ruby_system.network.master tcp_cntrl.responseToTCP = MessageBuffer(ordered = True) tcp_cntrl.responseToTCP.slave = ruby_system.network.master tcp_cntrl.mandatoryQueue = MessageBuffer() gpuCluster.add(tcp_cntrl) for i in xrange(options.num_sqc): sqc_cntrl = SQCCntrl(TCC_select_num_bits = TCC_bits) sqc_cntrl.create(options, ruby_system, system) exec("system.sqc_cntrl%d = sqc_cntrl" % i) # # Add controllers and sequencers to the appropriate lists # cpu_sequencers.append(sqc_cntrl.sequencer) # Connect the SQC controller to the ruby network sqc_cntrl.requestFromSQC = MessageBuffer(ordered = True) sqc_cntrl.requestFromSQC.master = ruby_system.network.slave sqc_cntrl.probeToSQC = MessageBuffer(ordered = True) sqc_cntrl.probeToSQC.slave = ruby_system.network.master sqc_cntrl.responseToSQC = MessageBuffer(ordered = True) sqc_cntrl.responseToSQC.slave = ruby_system.network.master sqc_cntrl.mandatoryQueue = MessageBuffer() # SQC also in GPU cluster gpuCluster.add(sqc_cntrl) numa_bit = 6 for i in xrange(options.num_tccs): tcc_cntrl = TCCCntrl() tcc_cntrl.create(options, ruby_system, system) tcc_cntrl.l2_request_latency = 1 tcc_cntrl.l2_response_latency = options.TCC_latency tcc_cntrl.WB = options.WB_L2 tcc_cntrl.number_of_TBEs = 2560 * options.num_compute_units # Connect the TCC controllers to the ruby network tcc_cntrl.requestFromTCP = MessageBuffer(ordered = True) tcc_cntrl.requestFromTCP.slave = ruby_system.network.master tcc_cntrl.responseToCore = MessageBuffer(ordered = True) tcc_cntrl.responseToCore.master = ruby_system.network.slave tcc_cntrl.probeFromNB = MessageBuffer() tcc_cntrl.probeFromNB.slave = ruby_system.network.master tcc_cntrl.responseFromNB = MessageBuffer() tcc_cntrl.responseFromNB.slave = ruby_system.network.master tcc_cntrl.requestToNB = MessageBuffer(ordered = True) tcc_cntrl.requestToNB.master = ruby_system.network.slave tcc_cntrl.responseToNB = MessageBuffer() tcc_cntrl.responseToNB.master = ruby_system.network.slave tcc_cntrl.unblockToNB = MessageBuffer() tcc_cntrl.unblockToNB.master = ruby_system.network.slave tcc_cntrl.triggerQueue = MessageBuffer(ordered = True) rb_cntrl = RBCntrl() rb_cntrl.create(options, ruby_system, system) rb_cntrl.number_of_TBEs = 2560 * options.num_compute_units rb_cntrl.isOnCPU = False # Connect the RB controllers to the ruby network rb_cntrl.requestFromCore = MessageBuffer(ordered = True) rb_cntrl.requestFromCore.slave = ruby_system.network.master rb_cntrl.responseFromCore = MessageBuffer() rb_cntrl.responseFromCore.slave = ruby_system.network.master rb_cntrl.requestToNetwork = MessageBuffer() rb_cntrl.requestToNetwork.master = ruby_system.network.slave rb_cntrl.notifyFromRegionDir = MessageBuffer() rb_cntrl.notifyFromRegionDir.slave = ruby_system.network.master rb_cntrl.probeFromRegionDir = MessageBuffer() rb_cntrl.probeFromRegionDir.slave = ruby_system.network.master rb_cntrl.unblockFromDir = MessageBuffer() rb_cntrl.unblockFromDir.slave = ruby_system.network.master rb_cntrl.responseToRegDir = MessageBuffer() rb_cntrl.responseToRegDir.master = ruby_system.network.slave rb_cntrl.triggerQueue = MessageBuffer(ordered = True) tcc_cntrl.regionBufferNum = rb_cntrl.version exec("system.tcc_cntrl%d = tcc_cntrl" % i) exec("system.tcc_rb_cntrl%d = rb_cntrl" % i) # TCC cntrls added to the GPU cluster gpuCluster.add(tcc_cntrl) gpuCluster.add(rb_cntrl) # Because of wire buffers, num_l3caches must equal num_dirs # Region coherence only works with 1 dir assert(options.num_l3caches == options.num_dirs == 1) # This is the base crossbar that connects the L3s, Dirs, and cpu/gpu # Clusters mainCluster = Cluster(intBW = crossbar_bw) dir_cntrl = DirCntrl() dir_cntrl.create(options, ruby_system, system) dir_cntrl.number_of_TBEs = 2560 * options.num_compute_units dir_cntrl.useL3OnWT = options.use_L3_on_WT # Connect the Directory controller to the ruby network dir_cntrl.requestFromCores = MessageBuffer() dir_cntrl.requestFromCores.slave = ruby_system.network.master dir_cntrl.responseFromCores = MessageBuffer() dir_cntrl.responseFromCores.slave = ruby_system.network.master dir_cntrl.unblockFromCores = MessageBuffer() dir_cntrl.unblockFromCores.slave = ruby_system.network.master dir_cntrl.probeToCore = MessageBuffer() dir_cntrl.probeToCore.master = ruby_system.network.slave dir_cntrl.responseToCore = MessageBuffer() dir_cntrl.responseToCore.master = ruby_system.network.slave dir_cntrl.reqFromRegBuf = MessageBuffer() dir_cntrl.reqFromRegBuf.slave = ruby_system.network.master dir_cntrl.reqToRegDir = MessageBuffer(ordered = True) dir_cntrl.reqToRegDir.master = ruby_system.network.slave dir_cntrl.reqFromRegDir = MessageBuffer(ordered = True) dir_cntrl.reqFromRegDir.slave = ruby_system.network.master dir_cntrl.unblockToRegDir = MessageBuffer() dir_cntrl.unblockToRegDir.master = ruby_system.network.slave dir_cntrl.triggerQueue = MessageBuffer(ordered = True) dir_cntrl.L3triggerQueue = MessageBuffer(ordered = True) dir_cntrl.responseFromMemory = MessageBuffer() exec("system.dir_cntrl%d = dir_cntrl" % i) dir_cntrl_nodes.append(dir_cntrl) mainCluster.add(dir_cntrl) reg_cntrl = RegionCntrl(noTCCdir=True,TCC_select_num_bits = TCC_bits) reg_cntrl.create(options, ruby_system, system) reg_cntrl.number_of_TBEs = options.num_tbes reg_cntrl.cpuRegionBufferNum = system.rb_cntrl0.version reg_cntrl.gpuRegionBufferNum = system.tcc_rb_cntrl0.version # Connect the Region Dir controllers to the ruby network reg_cntrl.requestToDir = MessageBuffer(ordered = True) reg_cntrl.requestToDir.master = ruby_system.network.slave reg_cntrl.notifyToRBuffer = MessageBuffer() reg_cntrl.notifyToRBuffer.master = ruby_system.network.slave reg_cntrl.probeToRBuffer = MessageBuffer() reg_cntrl.probeToRBuffer.master = ruby_system.network.slave reg_cntrl.responseFromRBuffer = MessageBuffer() reg_cntrl.responseFromRBuffer.slave = ruby_system.network.master reg_cntrl.requestFromRegBuf = MessageBuffer() reg_cntrl.requestFromRegBuf.slave = ruby_system.network.master reg_cntrl.triggerQueue = MessageBuffer(ordered = True) exec("system.reg_cntrl%d = reg_cntrl" % i) mainCluster.add(reg_cntrl) # Assuming no DMA devices assert(len(dma_devices) == 0) # Add cpu/gpu clusters to main cluster mainCluster.add(cpuCluster) mainCluster.add(gpuCluster) ruby_system.network.number_of_virtual_networks = 10 return (cpu_sequencers, dir_cntrl_nodes, mainCluster)
output["stdout"] = outs.decode("utf-8") output["stderr"] = errs.decode("utf-8") output["returncode"] = proc.returncode except (subprocess.TimeoutExpired) as _: Print( "ERROR: Nodpico is running beyond the defined wait time. Hard killing nodpico instance." ) proc.send_signal(signal.SIGKILL) return (False, None) if debug: Print("Returning success.") return (True, output) random.seed(seed) # Use a fixed seed for repeatability. cluster = Cluster(walletd=True) try: TestHelper.printSystemInfo("BEGIN") cluster.setChainStrategy(chainSyncStrategyStr) cluster.killall(allInstances=killAll) cluster.cleanup() Print( "producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % (pnodes, topo, delay, chainSyncStrategyStr)) Print("Stand up cluster") if cluster.launch(pnodes=pnodes,
if debug: Print("Setting nodeos process timeout.") outs,errs = proc.communicate(timeout=myTimeout) if debug: Print("Nodeos process has exited.") output["stdout"] = outs.decode("utf-8") output["stderr"] = errs.decode("utf-8") output["returncode"] = proc.returncode except (subprocess.TimeoutExpired) as _: Print("ERROR: Nodeos is running beyond the defined wait time. Hard killing nodeos instance.") proc.send_signal(signal.SIGKILL) return (False, None) if debug: Print("Returning success.") return (True, output) random.seed(seed) # Use a fixed seed for repeatability. cluster=Cluster(walletd=True) try: TestHelper.printSystemInfo("BEGIN") cluster.setChainStrategy(chainSyncStrategyStr) cluster.killall(allInstances=killAll) cluster.cleanup() Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % ( pnodes, topo, delay, chainSyncStrategyStr)) Print("Stand up cluster") if cluster.launch(pnodes, total_nodes, topo=topo, delay=delay, dontBootstrap=True) is False: errorExit("Failed to stand up eos cluster.")
# Keenan Albee and Antonio Teran from Cluster import Cluster import math import numpy as np import pandas as pd from cities import City, get_cities_table, parse_cities # Get the full city information as a table. # data = get_cities_table("1000-largest-us-cities.csv") # cities = parse_cities("1000-largest-us-cities.csv") # get an array with city objects. # for c in cities: # print(c) # Run a single city simulation dt = 0.1 # days sim_iters = int(math.floor(180 / dt)) # boston = Cluster(sim_iters, 10000, 1, 0) boston = Cluster(sim_iters, 4500000, 10000, 0) for i in range(sim_iters): boston.increment_model(dt) boston.plot_cluster()