def create_daemon(self, *args, **kwargs): self._daemon = DfmsDaemon(*args, **kwargs) if 'noNM' not in kwargs or not kwargs['noNM']: self.assertTrue(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, _TIMEOUT), 'The NM did not start successfully') if 'master' in kwargs and kwargs['master']: self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully') self._daemon_t = threading.Thread(target=lambda: self._daemon.start('localhost', 9000)) self._daemon_t.start() # Wait until the daemon's server has started # We can't simply check if the port is opened, because the server binds # before it is returned to us. In some tests we don't interact with it, # and therefore the shutdown of the daemon can occur before the server # is even returned to us. This would happen because portIsOpen will # succeed with a bound server, even if we haven't serve_forever()'d it # yet. In these situations shutting down the daemon will not shut down # the http server, and therefore the test will fail when checking that # the self._daemon_t is not alive anymore # # To actually avoid this we need to do some actual HTTP talk, which will # ensure the server is actually serving requests, and therefore already # in the daemon's hand #self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT)) try: restutils.RestClient('localhost', 9000, 10)._GET('/anything') except restutils.RestClientException: # We don't care about the result pass
def test_nothing_starts(self): # Nothing should start now self.create_daemon(master=False, noNM=True, disable_zeroconf=True) self.assertFalse( utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, 0), "NM started but it should not have" ) self.assertFalse( utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, 0), "NM started but it should not have" )
def _request(self, url, method, content=None, headers={}): # Do the HTTP stuff... if logger.isEnabledFor(logging.DEBUG): logger.debug("Sending %s request to %s:%d%s" % (method, self.host, self.port, url)) if not utils.portIsOpen(self.host, self.port, self.timeout): raise RestClientException("Cannot connect to %s:%d after %.2f [s]" % (self.host, self.port, self.timeout)) self._conn = httplib.HTTPConnection(self.host, self.port) self._conn.request(method, url, content, headers) self._resp = self._conn.getresponse() # Server errors are encoded in the body as json content if self._resp.status == httplib.INTERNAL_SERVER_ERROR: msg = json.loads(self._resp.read())['err_str'] if logger.isEnabledFor(logging.WARNING): logger.warning('Error found while requesting %s:%d%s: %s' % (self.host, self.port, url, msg)) raise RestClientException(msg) elif self._resp.status != httplib.OK: msg = 'Unexpected error while processing %s request for %s:%s%s (status %d): %s' % \ (method, self.host, self.port, url, self._resp.status, self._resp.read()) raise RestClientException(msg) return self._resp.read()
def test_start_master_via_rest(self): self.create_daemon(master=False, noNM=False, disable_zeroconf=True) # Check that the master starts self._start('master', httplib.OK) self.assertTrue(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), 'The MM did not start successfully')
def ensureDM(self, host, port=None, timeout=10): port = port or self._dmPort logger.debug("Checking DM presence at %s:%d", host, port) if portIsOpen(host, port, timeout): logger.debug("DM already present at %s:%d", host, port) return # We rely on having ssh keys for this, since we're using # the dfms.remote module, which authenticates using public keys logger.debug("DM not present at %s:%d, will start it now", host, port) self.startDM(host) # Wait a bit until the DM starts; if it doesn't we fail if not portIsOpen(host, port, timeout): raise DaliugeException( "DM started at %s:%d, but couldn't connect to it" % (host, port))
def test_start_master_via_rest(self): self.create_daemon(master=False, noNM=False, disable_zeroconf=True) # Check that the master starts self._start("master", httplib.OK) self.assertTrue( utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), "The MM did not start successfully", )
def ensureDM(self, host, timeout=10): # We rely on having ssh keys for this, since we're using # the dfms.remote module, which authenticates using public keys if logger.isEnabledFor(logging.DEBUG): logger.debug("Checking DM presence at %s:%d" % (host, self._dmPort)) if portIsOpen(host, self._dmPort, timeout): if logger.isEnabledFor(logging.DEBUG): logger.debug("DM already present at %s:%d" % (host, self._dmPort)) return if logger.isEnabledFor(logging.DEBUG): logger.debug("DM not present at %s:%d, will start it now" % (host, self._dmPort)) self.startDM(host) # Wait a bit until the DM starts; if it doesn't we fail if not portIsOpen(host, self._dmPort, timeout): raise Exception("DM started at %s:%d, but couldn't connect to it" % (host, self._dmPort))
def test_start_dataisland_via_rest(self): self.create_daemon(master=True, noNM=False, disable_zeroconf=False) # Both managers started fine. If they zeroconf themselves correctly then # if we query the MM it should know about its nodes, which should have # one element nodes = self._get_nodes_from_master(_TIMEOUT) self.assertIsNotNone(nodes) self.assertEqual(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node") # Check that the DataIsland starts with the given nodes self._start('dataisland', httplib.OK, {'nodes': nodes}) self.assertTrue(utils.portIsOpen('localhost', constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT), 'The DIM did not start successfully')
def create_daemon(self, *args, **kwargs): self._daemon = DfmsDaemon(*args, **kwargs) if "noNM" not in kwargs or not kwargs["noNM"]: self.assertTrue( utils.portIsOpen("localhost", constants.NODE_DEFAULT_REST_PORT, _TIMEOUT), "The NM did not start successfully", ) if "master" in kwargs and kwargs["master"]: self.assertTrue( utils.portIsOpen("localhost", constants.MASTER_DEFAULT_REST_PORT, _TIMEOUT), "The MM did not start successfully", ) self._daemon_t = threading.Thread(target=lambda: self._daemon.start("localhost", 9000)) self._daemon_t.start() # Wait until the daemon's server has started # We can't simply check if the port is opened, because the server binds # before it is returned to us. In some tests we don't interact with it, # and therefore the shutdown of the daemon can occur before the server # is even returned to us. This would happen because portIsOpen will # succeed with a bound server, even if we haven't serve_forever()'d it # yet. In these situations shutting down the daemon will not shut down # the http server, and therefore the test will fail when checking that # the self._daemon_t is not alive anymore # # To actually avoid this we need to do some actual HTTP talk, which will # ensure the server is actually serving requests, and therefore already # in the daemon's hand # self.assertTrue(utils.portIsOpen('localhost', 9000, _TIMEOUT)) try: restutils.RestClient("localhost", 9000, 10)._GET("/anything") except restutils.RestClientException: # We don't care about the result pass
def _request(self, url, method, content=None, headers={}): # Do the HTTP stuff... logger.debug("Sending %s request to %s:%d%s", method, self.host, self.port, url) if not utils.portIsOpen(self.host, self.port, self.timeout): raise RestClientException( "Cannot connect to %s:%d after %.2f [s]" % (self.host, self.port, self.timeout)) if content and hasattr(content, 'read'): headers['Transfer-Encoding'] = 'chunked' content = chunked(content) self._conn = httplib.HTTPConnection(self.host, self.port) self._conn.request(method, url, content, headers) self._resp = self._conn.getresponse() # Server errors are encoded in the body as json content if self._resp.status != httplib.OK: msg = 'Error on remote %s@%s:%s%s (status %d): ' % \ (method, self.host, self.port, url, self._resp.status) try: error = json.loads(self._resp.read().decode('utf-8')) etype = getattr(exceptions, error['type']) eargs = error['args'] if etype == SubManagerException: for host, args in eargs.items(): subetype = getattr(exceptions, args['type']) subargs = args['args'] eargs[host] = subetype(*subargs) ex = etype(eargs) else: ex = etype(*eargs) if hasattr(ex, 'msg'): ex.msg = msg + ex.msg except Exception: ex = RestClientException(msg + "Unknown") raise ex if not self._resp.length: return None return codecs.getreader('utf-8')(self._resp)
def test_start_dataisland_via_rest(self): self.create_daemon(master=True, noNM=False, disable_zeroconf=False) # Both managers started fine. If they zeroconf themselves correctly then # if we query the MM it should know about its nodes, which should have # one element nodes = self._get_nodes_from_master(_TIMEOUT) self.assertIsNotNone(nodes) self.assertEquals(1, len(nodes), "MasterManager didn't find the NodeManager running on the same node") # Check that the DataIsland starts with the given nodes self._start("dataisland", httplib.OK, {"nodes": nodes}) self.assertTrue( utils.portIsOpen("localhost", constants.ISLAND_DEFAULT_REST_PORT, _TIMEOUT), "The DIM did not start successfully", )
def check_host(host, port, timeout=5, check_with_session=False): """ Checks if a given host/port is up and running (i.e., it is open). If ``check_with_session`` is ``True`` then it is assumed that the host/port combination corresponds to a Node Manager and the check is performed by attempting to create and delete a session. """ if not check_with_session: return utils.portIsOpen(host, port, timeout) try: session_id = str(uuid.uuid4()) with NodeManagerClient(host, port, timeout=timeout) as c: c.create_session(session_id) c.destroy_session(session_id) return True except: return False
def main(): parser = optparse.OptionParser() parser.add_option("-l", "--log_dir", action="store", type="string", dest="log_dir", help="Log directory (required)") # if this parameter is present, it means we want to get monitored parser.add_option("-m", "--monitor_host", action="store", type="string", dest="monitor_host", help="Monitor host IP (optional)") parser.add_option("-o", "--monitor_port", action="store", type="int", dest="monitor_port", help="The port to bind dfms monitor", default=dfms_proxy.default_dfms_monitor_port) parser.add_option("-v", "--verbose-level", action="store", type="int", dest="verbose_level", help="Verbosity level (1-3) of the DIM/NM logging", default=1) parser.add_option( "-z", "--zerorun", action="store_true", dest="zerorun", help="Generate a physical graph that takes no time to run", default=False) parser.add_option( "--app", action="store", type="int", dest="app", help="The app to use in the PG. 1=SleepApp (default), 2=SleepAndCopy", default=0) parser.add_option( "-t", "--max-threads", action="store", type="int", dest="max_threads", help= "Max thread pool size used for executing drops. 0 (default) means no pool.", default=0) parser.add_option("-L", "--logical-graph", action="store", type="string", dest="logical_graph", help="The filename of the logical graph to deploy", default=None) parser.add_option( "-P", "--physical-graph", action="store", type="string", dest="physical_graph", help="The filename of the physical graph (template) to deploy", default=None) parser.add_option('-s', '--num_islands', action='store', type='int', dest='num_islands', default=1, help='The number of Data Islands') parser.add_option('-d', '--dump', action='store_true', dest='dump', help='dump file base name?', default=False) parser.add_option("-c", "--loc", action="store", type="string", dest="loc", help="deployment location (e.g. 'Pawsey' or 'Tianhe2')", default="Pawsey") parser.add_option("-u", "--all_nics", action="store_true", dest="all_nics", help="Listen on all NICs for a node manager", default=False) parser.add_option('--check-interfaces', action='store_true', dest='check_interfaces', help='Run a small network interfaces test and exit', default=False) parser.add_option( "-S", "--check_with_session", action="store_true", dest="check_with_session", help= "Check for node managers' availability by creating/destroy a session", default=False) (options, _) = parser.parse_args() if options.check_interfaces: print("From netifaces: %s" % utils.get_local_ip_addr()) print("From ifconfig: %s" % get_ip()) sys.exit(0) if options.logical_graph and options.physical_graph: parser.error( "Either a logical graph or physical graph filename must be specified" ) for p in (options.logical_graph, options.physical_graph): if p and not os.path.exists(p): parser.error("Cannot locate graph file at '{0}'".format(p)) if (options.monitor_host is not None and options.num_islands > 1): parser.error("We do not support proxy monitor multiple islands yet") logv = max(min(3, options.verbose_level), 1) from mpi4py import MPI # @UnresolvedImport comm = MPI.COMM_WORLD # @UndefinedVariable num_procs = comm.Get_size() rank = comm.Get_rank() log_dir = "{0}/{1}".format(options.log_dir, rank) os.makedirs(log_dir) logfile = log_dir + "/start_dfms_cluster.log" FORMAT = "%(asctime)-15s [%(levelname)5.5s] [%(threadName)15.15s] %(name)s#%(funcName)s:%(lineno)s %(message)s" logging.basicConfig(filename=logfile, level=logging.DEBUG, format=FORMAT) if (num_procs > 1 and options.monitor_host is not None): logger.info("Trying to start dfms_cluster with proxy") run_proxy = True threshold = 2 else: logger.info("Trying to start dfms_cluster without proxy") run_proxy = False threshold = 1 if (num_procs == threshold): logger.warning("No MPI processes left for running Drop Managers") run_node_mgr = False else: run_node_mgr = True # attach rank information at the end of IP address for multi-islands rank_str = '' if options.num_islands == 1 else ',%s' % rank public_ip = get_ip(options.loc) ip_adds = '{0}{1}'.format(public_ip, rank_str) origin_ip = ip_adds.split(',')[0] ip_adds = comm.gather(ip_adds, root=0) proxy_ip = None if run_proxy: # send island/master manager's IP address to the dfms proxy # also let island manager know dfms proxy's IP if rank == 0: mgr_ip = origin_ip comm.send(mgr_ip, dest=1) proxy_ip = comm.recv(source=1) elif rank == 1: mgr_ip = comm.recv(source=0) proxy_ip = origin_ip comm.send(proxy_ip, dest=0) set_env(rank) if (options.num_islands == 1): if (rank != 0): if (run_proxy and rank == 1): # Wait until the Island Manager is open if utils.portIsOpen(mgr_ip, ISLAND_DEFAULT_REST_PORT, 100): start_dfms_proxy(options.loc, mgr_ip, ISLAND_DEFAULT_REST_PORT, options.monitor_host, options.monitor_port) else: logger.warning( "Couldn't connect to the main drop manager, proxy not started" ) elif (run_node_mgr): logger.info( "Starting node manager on host {0}".format(origin_ip)) start_node_mgr(log_dir, logv=logv, max_threads=options.max_threads, host=None if options.all_nics else origin_ip) else: # 'no_nms' are known not to be NMs no_nms = [origin_ip, 'None'] if proxy_ip: no_nms += [proxy_ip] node_mgrs = [ip for ip in ip_adds if ip not in no_nms] # unroll the graph first (if any) while starting node managers on other nodes pgt = None if options.logical_graph or options.physical_graph: pip_name = utils.fname_to_pipname(options.logical_graph or options.physical_graph) if options.logical_graph: unrolled = tool.unroll(options.logical_graph, '1', options.zerorun, apps[options.app]) pgt = tool.partition(unrolled, pip_name, len(node_mgrs), options.num_islands, 'metis') del unrolled else: pgt = json.loads(options.physical_graph) # Check that which NMs are up and use only those form now on node_mgrs = check_hosts( node_mgrs, NODE_DEFAULT_REST_PORT, check_with_session=options.check_with_session, timeout=MM_WAIT_TIME) # We have a PGT, let's map it and submit it if pgt: pg = tool.resource_map(pgt, [origin_ip] + node_mgrs, pip_name, options.num_islands) del pgt def submit_and_monitor(): host, port = 'localhost', ISLAND_DEFAULT_REST_PORT tool.submit(host, port, pg) if options.dump: dump_path = '{0}/monitor'.format(log_dir) monitor_graph(host, port, dump_path) threading.Thread(target=submit_and_monitor).start() # Start the DIM logger.info("Starting island manager on host %s", origin_ip) start_dim(node_mgrs, log_dir, logv=logv) elif (options.num_islands > 1): if (rank == 0): # master manager # 1. use ip_adds to produce the physical graph ip_list = [] ip_rank_dict = dict() # k - ip, v - MPI rank for ipr in ip_adds: iprs = ipr.split(',') ip = iprs[0] r = iprs[1] if (ip == origin_ip or 'None' == ip): continue ip_list.append(ip) ip_rank_dict[ip] = int(r) if (len(ip_list) <= options.num_islands): raise Exception( "Insufficient nodes available for node managers") # 2 broadcast dim ranks to all nodes to let them know who is the DIM dim_ranks = [] dim_ip_list = ip_list[0:options.num_islands] logger.info("A list of DIM IPs: {0}".format(dim_ip_list)) for dim_ip in dim_ip_list: dim_ranks.append(ip_rank_dict[dim_ip]) dim_ranks = comm.bcast(dim_ranks, root=0) # 3 unroll the graph while waiting for node managers to start pip_name = utils.fname_to_pipname(options.logical_graph or options.physical_graph) if options.logical_graph: unrolled = tool.unroll(options.logical_graph, '1', options.zerorun, apps[options.app]) pgt = tool.partition(unrolled, pip_name, len(ip_list) - 1, options.num_islands, 'metis') del unrolled else: pgt = json.loads(options.physical_graph) #logger.info("Waiting all node managers to start in %f seconds", MM_WAIT_TIME) node_mgrs = check_hosts( ip_list[options.num_islands:], NODE_DEFAULT_REST_PORT, check_with_session=options.check_with_session, timeout=MM_WAIT_TIME) # 4. produce the physical graph based on the available node managers # that have already been running (we have to assume island manager # will run smoothly in the future) logger.info("Master Manager producing the physical graph") pg = tool.resource_map(pgt, dim_ip_list + node_mgrs, pip_name, options.num_islands) # 5. parse the pg_spec to get the mapping from islands to node list dim_rank_nodes_dict = collections.defaultdict(set) for drop in pg: dim_ip = drop['island'] # if (not dim_ip in dim_ip_list): # raise Exception("'{0}' node is not in island list {1}".format(dim_ip, dim_ip_list)) r = ip_rank_dict[dim_ip] n = drop['node'] dim_rank_nodes_dict[r].add(n) # 6 send a node list to each DIM so that it can start for dim_ip in dim_ip_list: r = ip_rank_dict[dim_ip] logger.debug("Sending node list to rank {0}".format(r)) #TODO this should be in a thread since it is blocking! comm.send(list(dim_rank_nodes_dict[r]), dest=r) # 7. make sure all DIMs are up running dim_ips_up = check_hosts(dim_ip_list, ISLAND_DEFAULT_REST_PORT, timeout=MM_WAIT_TIME, retry=10) if len(dim_ips_up) < len(dim_ip_list): logger.warning("Not all DIMs were up and running: %d/%d", len(dim_ips_up), len(dim_ip_list)) # 8. submit the graph in a thread (wait for mm to start) def submit(): if not check_host('localhost', MASTER_DEFAULT_REST_PORT, timeout=GRAPH_SUBMIT_WAIT_TIME): logger.warning( "Master Manager didn't come up in %d seconds", GRAPH_SUBMIT_WAIT_TIME) tool.submit('localhost', MASTER_DEFAULT_REST_PORT, pg) threading.Thread(target=submit).start() # 9. start dlgMM using islands IP addresses (this will block) start_mm(dim_ip_list, log_dir, logv=logv) else: dim_ranks = None dim_ranks = comm.bcast(dim_ranks, root=0) logger.debug("Receiving dim_ranks = {0}, my rank is {1}".format( dim_ranks, rank)) if (rank in dim_ranks): logger.debug( "Rank {0} is a DIM preparing for receiving".format(rank)) # island manager # get a list of nodes that are its children from rank 0 (MM) nm_list = comm.recv(source=0) # no need to wait for node managers since the master manager # has already made sure they are up running logger.debug("nm_list for DIM {0} is {1}".format( rank, nm_list)) start_dim(nm_list, log_dir, logv=logv) else: # node manager logger.info( "Starting node manager on host {0}".format(origin_ip)) start_node_mgr(log_dir, logv=logv, max_threads=options.max_threads, host=None if options.all_nics else origin_ip)
def tearDown(self): unittest.TestCase.tearDown(self) self._daemon.stop(_TIMEOUT) self._daemon_t.join(_TIMEOUT) self.assertFalse(self._daemon_t.is_alive(), "Daemon running thread should have finished by now") self.assertFalse(utils.portIsOpen("localhost", 9000, 0), "DFMS Daemon REST interface should be off")
def test_nothing_starts(self): # Nothing should start now self.create_daemon(master=False, noNM=True, disable_zeroconf=True) self.assertFalse(utils.portIsOpen('localhost', constants.NODE_DEFAULT_REST_PORT, 0), 'NM started but it should not have') self.assertFalse(utils.portIsOpen('localhost', constants.MASTER_DEFAULT_REST_PORT, 0), 'NM started but it should not have')
def test_fullRound(self): """ A test that exercises most of the REST interface exposed on top of the NodeManager """ sessionId = 'lala' restPort = 8888 args = [sys.executable, '-m', 'dfms.manager.cmdline', 'dfmsNM', \ '--port', str(restPort), '-qqq'] dmProcess = subprocess.Popen(args) with testutils.terminating(dmProcess, 10): # Wait until the REST server becomes alive self.assertTrue(utils.portIsOpen('localhost', restPort, 10), "REST server didn't come up in time") # The DM is still empty dmInfo = testutils.get(self, '', restPort) self.assertEquals(0, len(dmInfo['sessions'])) # Create a session and check it exists testutils.post(self, '/sessions', restPort, '{"sessionId":"%s"}' % (sessionId)) dmInfo = testutils.get(self, '', restPort) self.assertEquals(1, len(dmInfo['sessions'])) self.assertEquals(sessionId, dmInfo['sessions'][0]['sessionId']) self.assertEquals(SessionStates.PRISTINE, dmInfo['sessions'][0]['status']) # Add this complex graph spec to the session # The UID of the two leaf nodes of this complex.js graph are T and S # PRO-242: use timestamps for final DROPs that get archived into the public NGAS graph = json.loads(pkg_resources.resource_string('test', 'graphs/complex.js')) # @UndefinedVariable suffix = '_' + str(int(time.time())) oidsToReplace = ('S','T') for dropSpec in graph: if dropSpec['oid'] in oidsToReplace: dropSpec['oid'] += suffix for rel in ('inputs','outputs'): if rel in dropSpec: for oid in dropSpec[rel][:]: if oid in oidsToReplace: dropSpec[rel].remove(oid) dropSpec[rel].append(oid + suffix) testutils.post(self, '/sessions/%s/graph/append' % (sessionId), restPort, json.dumps(graph)) # We create two final archiving nodes, but this time from a template # available on the server-side timeout = 10 testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving1&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort) testutils.post(self, '/templates/dfms.manager.repository.archiving_app/materialize?uid=archiving2&host=ngas.ddns.net&port=7777&sessionId=%s&connect_timeout=%f&timeout=%f' % (sessionId, timeout, timeout), restPort) # And link them to the leaf nodes of the complex graph testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving1&lhOID=S%s&linkType=0' % (sessionId, suffix), restPort) testutils.post(self, '/sessions/%s/graph/link?rhOID=archiving2&lhOID=T%s&linkType=0' % (sessionId, suffix), restPort) # Now we deploy the graph... testutils.post(self, '/sessions/%s/deploy' % (sessionId), restPort, 'completed=SL_A,SL_B,SL_C,SL_D,SL_K', mimeType='application/x-www-form-urlencoded') # ...and write to all 5 root nodes that are listening in ports # starting at 1111 msg = ''.join([random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in xrange(10)]) for i in xrange(5): self.assertTrue(utils.writeToRemotePort('localhost', 1111+i, msg, 2), "Couldn't write data to localhost:%d" % (1111+i)) # Wait until the graph has finished its execution. We'll know # it finished by polling the status of the session while testutils.get(self, '/sessions/%s/status' % (sessionId), restPort) == SessionStates.RUNNING: time.sleep(0.2) self.assertEquals(SessionStates.FINISHED, testutils.get(self, '/sessions/%s/status' % (sessionId), restPort)) testutils.delete(self, '/sessions/%s' % (sessionId), restPort) # We put an NGAS archiving at the end of the chain, let's check that the DROPs were copied over there # Since the graph consists on several SleepAndCopy apps, T should contain the message repeated # 9 times, and S should have it 4 times def checkReplica(dropId, copies): response = ngaslite.retrieve('ngas.ddns.net', dropId) buff = response.read() self.assertEquals(msg*copies, buff, "%s's replica doesn't look correct" % (dropId)) checkReplica('T%s' % (suffix), 9) checkReplica('S%s' % (suffix), 4)