def main(agrv=None): start = time.time() is_training = tf.placeholder(tf.bool, name='is_train') #BN: istraining image = tf.placeholder(tf.float32, shape=[None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name='input_img') # logits, pred = net.u_net(image, is_training) logits, pred = net.PSPUnet(image, is_training) print('setup session...') print("Setting up dataset reader") config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) print("Setting up Saver...") saver = tf.train.Saver(max_to_keep=50) sess.run(tf.global_variables_initializer()) saver.restore(sess, modelname) img_folder = img_path test_itr = 0 for imgname in os.listdir(img_folder): test_itr = test_itr + 1 print('path is :', img_folder + imgname) test_img_batch = testutils.read_img(img_folder + imgname) test_img_sp = data_pro.image_splite(test_img_batch, COLS, ROWS) pred_sp = {} logits_sp = {} for col in range(COLS): pred_sp[col] = {} logits_sp[col] = {} for row in range(ROWS): img_test = test_img_sp[col][row] test_feed = {image: img_test, is_training: False} test_pred_logits, pred_image = sess.run([logits, pred], feed_dict=test_feed) logits_sp[col][row] = test_pred_logits pred_sp[col][row] = pred_image pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1) pred_batch = np.squeeze(pred_all) print("test_itr:", test_itr) testutils.saveImage(img_folder + imgname, pred_batch, imgname, test_save_path) # pred_batch = np.squeeze(pred_all) # pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH]) # print("test_itr:",test_itr) # utils.save_Test(imgname[:-4], pred_tosave, test_save_path) end = time.time() elapse = end - start print('elapse time is :', elapse)
def testNodeStartup(self): # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost') self.assertTrue(status, 'Could not build bootstrap node') # Start a client node (status, self.normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346, 'localhost', self.bsNode.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') # Are they connected together? status = yield self.doQuickSendTest() self.assertTrue(status, 'doQuickSendTest') # Stop the nodes yield self.normalNode.leave() yield self.bsNode.leave() # Wait for the connections to really all close if Config.USE_CONNECTION_CACHE: yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3) else: yield TestUtils.wait(3) defer.returnValue(True)
def testNodeStartup(self): port = 12345 enclave = 'localhost' bootstrapNodeLocation = NodeLocation(None, self.myIP, port) # Start a bootstrap node (status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave) self.assertTrue(status, 'Could not build bootstrap node') # Start a client node (status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation) self.assertTrue(status, 'Could not startupClientNode') #yield TestUtils.wait(3) # Are they connected together? status = yield self.doConnectionTest(node) self.assertTrue(status, 'doConnectionTest') # Stop the nodes yield bsNode.leave() yield node.leave() yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3) defer.returnValue(True)
def getAggResponse(self): '''Send a query and get an agg response from all the nodes.''' bsNode = self.bsNetwork.chordNode for aggNumber in range(1, numNodes + 5): # Reset the message counter obs = self.getObserver(bsNode, self.allTestObservers) obs.resetMessageCount() # Send the query asking for a response messageNum = aggNumber # Just to keep it incrementing yield TestUtils.sendFlood(bsNode,messageNum,'localhost', data="SEND_AGG_RESPONSE:%d" % aggNumber) # Now check how many messages the bootstrap node got in return for _ in range(10): # 10 seconds recvCount = obs.getMessageCount() if recvCount >= numNodes+1: break else: yield TestUtils.wait(1) self.failUnless(recvCount == numNodes+1, "getAggResponse didn't get the correct number of messages back.Expected[%d] Got[%d]" % (numNodes+1, recvCount)) defer.returnValue(True)
def testSerialP2PSending(self): # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost') self.assertTrue(status, 'Could not build bootstrap node') self.allNodes.append(self.bsNode) self.bsNode.addMessageObserver(self.messageReceived) # Start client nodes log.msg("Building nodes...") for i in range(numNodes): (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') self.allNodes.append(node) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself. # Do the real test status = yield self.doStressTest() # Now close it all down! yield self.allLeave() # Wait a second or two yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY) defer.returnValue(True)
def test_iter_and_next(self): lst = [('age', 23), ('score', 95), ('color', 'blue')] dictionary = Dictionary() dictionary.from_list(lst) tmp_1 = [] for key, value in dictionary: tmp_1.append((key, value)) sorted_tmp = TestUtils.sort(tmp_1) t = dictionary.to_list() self.assertEqual(t, sorted_tmp) # Test for two iterators on one data structure working in parallel. tmp_2 = [] tmp_3 = [('age', 'age'), ('age', 'score'), ('age', 'color'), ('score', 'age'), ('score', 'score'), ('score', 'color'), ('color', 'age'), ('color', 'score'), ('color', 'color')] for iterator_1 in dictionary: for iterator_2 in dictionary: tmp_2.append((iterator_1[0], iterator_2[0])) tmp_2 = TestUtils.sort(tmp_2) tmp_3 = TestUtils.sort(tmp_3) self.assertEqual(tmp_2, tmp_3) it = iter(Dictionary()) self.assertRaises(StopIteration, lambda: next(it))
def test_add(self): dictionary = Dictionary() lst = [] dictionary.add("score", 89) lst.append(("score", 89)) self.assertEqual(dictionary.to_list(), [("score", 89)]) dictionary.add("score", 78) lst.append(("score", 78)) self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) dictionary.add("gender", "male") lst.append(("gender", "male")) self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) dictionary2 = Dictionary() temp1 = (23, 34) temp2 = (23, 34) dictionary2.add(temp1, 'test_value_1') dictionary2.add(temp2, 'value2') lst1 = [(temp2, 'value2'), (temp1, 'test_value_1')] self.assertEqual(dictionary2.to_list(), TestUtils.sort(lst1)) # Exception test # 1. Add repeated key-value. dictionary.add("score", 78) self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) # 2. Add invalid key self.assertRaises(Exception, dictionary.add, { 'key1': 2, "key2": 3 }, "dict_key_test") # 3. Add None value self.assertRaises(Exception, dictionary.add, "score", None)
def testSerialFlooding(self): # Start a bootstrap node (status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost') self.assertTrue(status, 'Could not build bootstrap node') self.allNodes.append(self.bsNode) self.bsNode.addMessageObserver(self.messageReceived) # Start client nodes for i in range(numNodes): (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') self.allNodes.append(node) observer = MyMessageObserver() node.addMessageObserver(observer.messageReceived) observer.node = node self.allObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself. # Now do the stress test status = yield self.doStressTest() # Now close it all down! yield self.allLeave() # Wait a second or two for network timeouts yield TestUtils.wait(9) defer.returnValue(True)
def test_associativity(self, lst_1, lst_2, lst_3): lst_1 = TestUtils.lst_validate(lst_1) lst_2 = TestUtils.lst_validate(lst_2) lst_3 = TestUtils.lst_validate(lst_3) dictionary_1 = Dictionary() dictionary_1.from_list(lst_1) dictionary_2 = Dictionary() dictionary_2.from_list(lst_2) dictionary_3 = Dictionary() dictionary_3.from_list(lst_3) # It equals to "(ab)c". dictionary_1.mconcat(dictionary_2) dictionary_1.mconcat(dictionary_3) temp1 = dictionary_1 dictionary_1 = Dictionary() dictionary_1.from_list(lst_1) dictionary_2 = Dictionary() dictionary_2.from_list(lst_2) # It equals to "a(bc)". dictionary_2.mconcat(dictionary_3) dictionary_1.mconcat(dictionary_2) temp2 = dictionary_1 assert id(temp2) != id(temp1) # To determine if "(ab)c = a(bc)" holds. self.assertEqual(temp1, temp2)
def sendP2PMsg(self, src, dst, allNodeObservers): '''Send a P2P message from src to dst and verify it got there. Also check that no other nodes got it. ''' goodCounter = 0 for _ in range(numMessages): messageNum = random.randint(1,9999999) # Send the message status = yield TestUtils.sendP2P(src, dst, messageNum ) self.assertTrue(status,"sendP2PMsg failed! [%s] [%s] to [%s]" % (status, src, dst)) #yield TestUtils.wait(1) # Wait for the messages to send # Check receipt (both that one did and others did not) status = TestUtils.didNodeReceive(allNodeObservers,dst, messageNum) self.assertTrue(status,"Expected node did not receive the message!") status = TestUtils.didNotReceive(allNodeObservers, 'ALL', messageNum, len(allNodeObservers) - 1) self.assertTrue(status,"Extra nodes received message!") goodCounter += 1 self.assertTrue(goodCounter == numMessages, "sendP2P message did not finish all messages [%d][%d]" % (goodCounter, numMessages)) defer.returnValue(True)
def test_copy_force(self): testFileName = "test_file.txt" data = { "target": "{0}/{1}".format(TestUtils.CreateAndGetTMPDir(), testFileName), "to": "{0}/tmp".format(TestUtils.CreateAndGetTMPDir()), "force": True } with open(data["target"], 'w+') as fileObj: fileObj.write("1") resFile = "{0}/{1}".format(data["to"], testFileName) with open(resFile, 'w+') as fileObj: fileObj.write("2") step = CopyFile() step.serialize(data) self.assertTrue(step.run()) self.assertTrue(os.path.exists(resFile)) with open(resFile, 'r') as fileObj: data = fileObj.read() self.assertEqual(data, "2")
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] enclave = "theEnc" # Start a bootstrap node (status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave) self.assertTrue(status, "Could not build bootstrap node") self.allBootstrapObservers.append(observer) # Add X nodes to each enclave for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode( self.myIP, startingPort, enclave, bootstrapNodeLocation ) self.assertTrue(status, "Could not startupClientNode") startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes + 1, self.bsNode) # Does not bsNode . defer.returnValue(True)
def createOneWrongArray(size, count): arrays[size] = [] for i in range(count): random.seed(i) array = list(range(size)) if size > 1: a = b = -1 while a == b: a = random.randint(0, size - 1) b = random.randint(0, size - 1) TestUtils.swap(array, a, b) arrays[size].append(array)
def test_map_my(self): lst = [('score', [98, {99, 100}]), ('age', 23), ('length', 50)] dictionary = Dictionary() dictionary.from_list(lst) dictionary.map_my(lambda x: x + 1) result = [('age', 24), ('length', 51), ('score', [99, [100, 101]])] self.assertEqual(dictionary.to_list(), TestUtils.sort(result))
def testStartAutoDiscoveryClientTimeout(self): '''Try to start a bootstrap node with autodiscovery on. This should show a failure notice. ''' log.msg("---------------------- BEGIN testStartAutoDiscoveryClientTimeout -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Create Bootstrap port = 12345 enclaveStr = 'testclave' # Now try and join with autodiscovery d = defer.Deferred() clientAPI = SampleClient(self.myIP, port, None) networkAPI = classChordNetworkChord(clientAPI, port, self.myIP) nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests # Join the network callFunc = lambda result, payload: self.shouldFailCallback(result, payload, d) networkAPI.start(callFunc, nodeID, enclaveStr, "authenticate:succeed", None, False, True) yield d yield TestUtils.waitForConnectionCache()
def getTestSuite(select="unit"): """ Get test suite select is one of the following: "unit" return suite of unit tests only "component" return suite of unit and component tests "all" return suite of unit, component and integration tests "pending" return suite of pending tests name a single named test to be run """ testdict = { "unit": [ "testUnits" , "testNull" ], "component": [ "testComponents" ], "integration": [ "testIntegration" ], "pending": [ "testPending" ] } return TestUtils.getTestSuite(SparqlQueryTestCase, testdict, select=select)
def testMultipleBSNodesAutoDiscoveryClient(self): '''Try to start multiple bootstrap nodes then a client and make sure it connects to one of them. ''' log.msg("---------------------- BEGIN testMultipleBSNodesAutoDiscoveryClient -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Create Bootstrap port = 12345 enclaveStr = 'testclave' bootstrapNodeList = [ NodeLocation(None, self.myIP, port), NodeLocation(None, self.myIP, port+1)] (self.bsClient1, self.bsNetwork1, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList) yield d (self.bsClient2, self.bsNetwork2, d) = self.startBootstrapNode(enclaveStr, self.myIP, port+1, "authenticate:succeed", None) yield d # Now try and join with autodiscovery yield self.startClientNodeAutoDiscovery(None, self.myIP, port+2, enclaveStr, bootstrapNodeList) # Shut it all down for (clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork1.disconnect() yield self.bsNetwork2.disconnect() yield TestUtils.waitForConnectionCache()
def getTestSuite(select="unit"): """ Get test suite select is one of the following: "unit" return suite of unit tests only "component" return suite of unit and component tests "all" return suite of unit, component and integration tests "pending" return suite of pending tests name a single named test to be run """ testdict = { "unit": [ "oai_listIdentifiers" ], "component": [ "testComponents" ], "integration": [ "testIntegration" ], "pending": [ "testPending" ] } return TestUtils.getTestSuite(TestOaiClient, testdict, select=select)
def waitForAllReceived(self): '''Wait until all messages have been received by all the nodes.''' numTries = 10 expectedNumMsg = numNodes*numMessages for _ in range(numTries): completedObservers = 0 incompleteObservers = 0 # Wait a sec yield TestUtils.wait(1) # Count them for obs in self.allObservers: numRx = obs.getNumReceived() if numRx == expectedNumMsg: completedObservers += 1 elif numRx > expectedNumMsg: obs.printMessages() raise Exception("Programming error... received more messages than possible! Got[%d] Expected[%d]" % (numRx,expectedNumMsg )) else: incompleteObservers += 1 print("waitForAllReceived: Complete:%d Incomplete:%d" % (completedObservers, incompleteObservers)) if incompleteObservers == 0: defer.returnValue(True) defer.returnValue(False)
def filter_files(files): '''filter list of files according to filters set in configuration file tests/_test_commandline.yml''' # directory location of tests testing_dir = TestUtils.get_tests_directory() # the config file config_file = os.path.join(testing_dir, "_test_commandline.yml") if os.path.exists(config_file): config = yaml.load(open(config_file)) if config is not None: if "restrict" in config and config["restrict"]: values = config["restrict"] if "manifest" in values: # take scripts defined in the MANIFEST.in file scriptdirs = [ x for x in open("MANIFEST.in") if x.startswith("include CGAT/tools") and x.endswith(".py\n") ] take = set( [re.sub("include\s*", "", x[:-1]) for x in scriptdirs]) files = [x for x in files if x in take] if "regex" in values: rx = re.compile(values["regex"]) files = filter(rx.search, files) return files
def testStartAutoDiscoveryClient(self): '''Try to start a bootstrap node with autodiscovery on. ''' log.msg("---------------------- BEGIN testStartAutoDiscoveryClient -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Create Bootstrap port = 12345 enclaveStr = 'testclave' bsNodeLocation = NodeLocation(None, self.myIP, port) bootstrapNodeList = [ bsNodeLocation ] (self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList) yield d # Now try and join with autodiscovery yield self.startClientNodeAutoDiscovery(None, self.myIP, port+1, enclaveStr, bootstrapNodeList) # Shut it all down for (clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() # Try and wait for connection cache to finish disconnects (just in case). yield TestUtils.waitForConnectionCache()
def waitForConnectivity(self, numToWaitFor, chordNode): '''Wait till we can connect to all numNodes''' self.node = chordNode self.testCounter = 1 self.connectedNodeList = [] # Need to see the messages self.node.addMessageObserver(self.messageReceived) yield self.getNumConnected() while len(self.connectedNodeList) < numToWaitFor: log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter") self.testCounter += 1 yield self.getNumConnected() yield TestUtils.wait(5) # Wait for messages to go around log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter") # for n in self.connectedNodeList: # print("DEBUG: Node:%s" % n) # Don't care anymore self.node.removeMessageObserver(self.messageReceived)
def test_copy_invalid_file(self): testFileName = "test_file.txt" data = { "target": "{0}/{1}".format(TestUtils.CreateAndGetTMPDir(), testFileName), "to": "{0}/tmp".format(TestUtils.CreateAndGetTMPDir()), "force": False } self.assertFalse(os.path.exists(data["target"])) resFile = "{0}/{1}".format(data["to"], testFileName) step = CopyFile() step.serialize(data) self.assertFalse(step.run()) self.assertFalse(os.path.exists(resFile))
def test_from_list(self): lst = [('name', 'Nick'), ('age', 23), ('gender', 'male'), ('others', [10, 100])] lst_2 = [('name', 'Nick'), ('others', [10, 100, 200])] dictionary = Dictionary() dictionary.from_list(lst) self.assertEqual(dictionary.get_by_key('gender'), 'male') self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) # Existing dictionary object calls from_lst(). dictionary.from_list(lst_2) self.assertEqual(dictionary.to_list(), TestUtils.sort(lst_2)) # Exception test # There are invalid key or value in the list. lst_3 = [('name', None), ('others', [10, 100, 200]), (None, '10')] self.assertRaises(Exception, dictionary.from_list, lst_3)
def doTest(): TestDataCube.doTest() TestShape.doTest() TestSlice.doTest() TestTimer.doTest() TestUtils.doTest() TestDtypeInfo.doTest() TestNdArray.doTest() TestMethods.doTest() TestCoordinates.doTest() TestConstants.doTest() TestLinalg.doTest() TestRandom.doTest() TestRotations.doTest() TestFilters.doTest() TestPolynomial.doTest() TestFFT.doTest() TestImageProcessing.doTest()
def waitForMessageCount(self, obs, numMessagesToWaitFor): for _ in range(10): if obs.getPingbackCount() >= numMessagesToWaitFor: defer.returnValue(True) else: yield TestUtils.wait(1) defer.returnValue(False)
def buildNetwork(self): global startingPort # Create two Bootstrap nodes port = 12345 self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] # Start a bootstrap node (status, self.bsNode1, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Start another bootstrap node (status, self.bsNode2, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port+1, allEnclaves[1]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Add some nodes to each bootstrap node for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[0], self.bsNode1.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) for _ in range(numNodes): # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[1], self.bsNode2.nodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes, self.bsNode1) # Does not count bsNode itself. yield waiter.waitForConnectivity(numNodes, self.bsNode2) # Does not count bsNode itself. defer.returnValue(True)
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allNetworkAPIs = [] self.allClientAPIs = [] # Start a bootstrap node log.msg("building BS node...") (status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True) self.assertTrue(status, 'Could not build bootstrap node') node = networkAPI.chordNode # This is very highly coupled -- bad MetricsMessageObserver(node) self.observer = TestMessageObserver(node, networkAPI) self.bsNode = node log.msg("building client nodes...") # Add X nodes to each enclave for _ in range(numNodes): # Add a node to the enclave (status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, startingPort, bootstrapNodeLocation, 'theEnclave', False, False) self.assertTrue(status, 'Could not startupClientNode') node = networkAPI.chordNode # This is very highly coupled -- bad MetricsMessageObserver(node) observer = TestMessageObserver(node, networkAPI) startingPort += 1 self.allNetworkAPIs.append(networkAPI) self.allNodeObservers.append(observer) self.allClientAPIs.append(clientAPI) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(numNodes+1, self.bsNode) # Does count bsNode itself. defer.returnValue(True)
def doTearDown(self): '''Tear down the network created during setup.''' log.msg("tearDown begins...") # Stop everything for networkAPI in self.allNetworkAPIs: yield networkAPI.disconnect() yield self.bsNode.leave() # Wait for all network timeouts to finish #self.flushLoggedErrors() if Config.USE_CONNECTION_CACHE: yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 2) else: yield TestUtils.defWait(3) defer.returnValue(True)
def test_set_value(self): dictionary = Dictionary() dictionary.add("age", 23) dictionary.set_value("age", 100) self.assertEqual(dictionary.get_by_key("age"), 100) # Set a value to a key which not exist. dictionary.set_value("color", "blue") self.assertEqual(dictionary.to_list(), TestUtils.sort([("age", 100), ("color", "blue")]))
def testAggregationMessageAPI(self): ''' By the time this function is called, we should have a full network built and connected. Tests to perform now: 1. Send P2P messages from BS Node to all nodes 2. Send P2P messages from all Nodes to BSNode 3. Send flooding from BSNode to all 4. Send query and return aggregation responses of different levels Finally, do cleanup. ''' try: # Build a network yield self.buildNetwork() # 1. Send P2P messages from BS Node to all nodes yield self.p2pFromBStoAll() # 2. Send P2P messages from all Nodes to BSNode yield self.p2pFromAlltoBS() # 3. Send flooding from BSNode to all yield self.floodFromBStoAll() # 4. Send query and return aggregation responses of different levels yield self.getAggResponse() # Do the cleanup yield TestUtils.wait(5) # Shut it all down for (_clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() # Now wait for the network cache disconnect timeout yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 5) except Exception, e: log.err(e, "An error occurred in testAggregationMessageAPI")
def test_remove_key(self): dictionary = Dictionary() dictionary.add("name", "Nick") dictionary.add("age", 23) dictionary.add("gender", "male") dictionary.add("score", 10) dictionary.add("score", 100) dictionary.remove_key("gender") lst = [("age", 23), ("name", "Nick"), ("score", 10), ("score", 100)] self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) dictionary.remove_key("score") lst = [("age", 23), ("name", "Nick")] self.assertEqual(dictionary.to_list(), TestUtils.sort(lst)) # Exception test # 1. Remove elements that do not exist. self.assertRaises(Exception, dictionary.remove_key, 23) # 2. Remove None key. self.assertRaises(Exception, dictionary.remove_key, None)
def test_to_list(self): dictionary = Dictionary() self.assertEqual(dictionary.to_list(), []) dictionary.add("name", "Nick") dictionary.add("age", 23) dictionary.add("gender", "male") dictionary.add("others", 10) dictionary.add("others", 100) lst = [("age", 23), ("gender", "male"), ("name", "Nick"), ("others", 10), ("others", 100)] self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
def __init__(self, tag, name="scimma-tst-srv", net="scimma-test", opts=[], dbg=False): super(ServerContainer, self).__init__(name=name) self.tag = tag self.cimage = tu.clientImage(tag) self.simage = tu.serverImage(tag) self.name = name self.options = opts self.network = net self.q = multiprocessing.Queue() self.cmd = "docker" self.args = [ "run", "--detach=true", "--rm=true", "--network=%s" % net, "-v", "shared:/root/shared", "--name=%s" % name ] + opts + [self.simage]
def doTest(): TestConstants.doTest() TestCoordinates.doTest() TestDataCube.doTest() TestDtypeInfo.doTest() TestFilters.doTest() TestFunctions.doTest() TestImageProcessing.doTest() TestIntegrate.doTest() TestLinalg.doTest() TestNdArray.doTest() TestPolynomial.doTest() TestRandom.doTest() TestRotations.doTest() TestRoots.doTest() TestShape.doTest() TestSlice.doTest() TestSpecial.doTest() TestTimer.doTest() TestUtils.doTest() TestVector.doTest()
def buildNetwork(self): global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodeObservers = [] self.allBootstrapObservers = [] self.allNodes = [] # Start a bootstrap node log.msg("---> Startup Bootstrap Node", system="EnclaveP2P Test") (status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0]) self.assertTrue(status, 'Could not build bootstrap node') self.allBootstrapObservers.append(observer) # Join another enclave enableAutoDiscovery = True bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ] status = yield self.bsNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1]) # Add X nodes to each enclave for enclave in allEnclaves: for _ in range(numNodes): log.msg("---> Add node to enclave", system="EnclaveP2P Test") # Add a node to the enclave (status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, enclave, bootstrapNodeLocation) self.assertTrue(status, 'Could not startupClientNode') startingPort += 1 self.allNodes.append(node) self.allNodeObservers.append(observer) # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() numberOfNodes = (len(allEnclaves) * numNodes) yield waiter.waitForConnectivity(numberOfNodes+1, self.bsNode) # Does not count bsNode itself. defer.returnValue(True)
def testStartAutoDiscoverySameNodeError(self): '''Try to start a bootstrap node and then two clients on the same port with autodiscovery on. Should log an error when the second node attempts to join. ''' log.msg("---------------------- BEGIN testStartAutoDiscoverySameNodeError -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Create Bootstrap port = 12345 enclaveStr = 'testclave' bsNodeLocation = NodeLocation(None, self.myIP, port) bootstrapNodeList = [ bsNodeLocation ] (self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList) yield d # Now startup one client which will succeed d2 = defer.Deferred() port = port + 1 clientAPI = SampleClient(self.myIP, port, None) networkAPI = classChordNetworkChord(clientAPI, port, self.myIP) nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests # Join the network callFunc = lambda result, payload: self.shouldSucceedCallback(result, payload, d2, networkAPI) networkAPI.start(callFunc, nodeID, enclaveStr, "authenticate:succeed", bootstrapNodeList, False, True) yield d2 # Now startup one which should fail due to duplicate port d3 = defer.Deferred() clientAPI2 = SampleClient(self.myIP, port, None) networkAPI2 = classChordNetworkChord(clientAPI2, port, self.myIP) nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests # Join the network callFunc2 = lambda result, payload: self.shouldFailureCallback(result, payload, d3) networkAPI2.start(callFunc2, nodeID, enclaveStr, "authenticate:succeed",bootstrapNodeList, False, True) yield d3 yield networkAPI.disconnect() #yield networkAPI2.disconnect() yield self.bsNetwork.disconnect() # Try and wait for connection cache to finish disconnects (just in case). yield TestUtils.waitForConnectionCache()
def testNodeAuth(self): log.msg("---> Starting BootstrapNode") yield self.startBootStrapNode() log.msg("---> Creating Good Client") yield self.startClientNode(self.myIP, 12350, self.bootstrapNodeLocation, self.enclaveStr) log.msg("---> Creating Bad Client") yield self.startBadClientNode(self.myIP, 12351, self.bootstrapNodeLocation, self.enclaveStr) # Shut it all down for (clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() if Config.USE_CONNECTION_CACHE: yield TestUtils.waitForConnectionCache() else: yield TestUtils.wait(2) defer.returnValue(True)
def testAutoDiscoveryRebootstrap(self): '''Try to start a bootstrap node and client with autodiscovery on. Then start another bootstrap node. Then kill first bootstrap node and have client try to re-bootstrap to new bootstrap node. ''' log.msg("---------------------- BEGIN testAutoDiscoveryRebootstrap -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Create Bootstrap port = 12345 enclaveStr = 'testclave' bsNodeLocation = NodeLocation(None, self.myIP, port) bootstrapNodeList = [ bsNodeLocation ] log.msg("Starting bootstrap... \n\n") (self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList) yield d # Now try and join with autodiscovery yield TestUtils.wait(1) log.msg("Starting client... \n\n") yield self.startClientNodeAutoDiscovery(None, self.myIP, port+1, enclaveStr, bootstrapNodeList) # Now start a second bootstrap node #NOT DONE YET log.msg("Shutting down client... \n\n") # Shut it all down for (clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() # Try and wait for connection cache to finish disconnects (just in case). yield TestUtils.waitForConnectionCache()
def sendFailingP2PMsg(self, src, dst): '''Send a P2P message from src to dst that should fail to get there because they are in different enclaves. Also check that no other nodes got it. ''' for _ in range(numMessages): messageNum = random.randint(1,9999999) # Send the message status = yield TestUtils.sendP2P(src, dst, messageNum ) self.assertFalse(status,"sendP2PMsg succeeded but should have failed! [%s] to [%s]" % (src, dst)) defer.returnValue(True)
def testEnclaveBridge(self): '''Create a node that can bridge between the enclaves Send a flooding message from it to 1 enclave and verify only nodes in that one got it ''' authenticationPayload = "This is a test" # Build the bridge node (status, bridgeNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12500, allEnclaves[0], self.bsNode1.nodeLocation) self.failUnless(status, 'testEnclave bridge could not join first bootstrap [%s]' % status) # Join the other enclave also. enableAutoDiscovery = True bootstrapNodeList = [ self.bsNode2.nodeLocation ] status = yield bridgeNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, authenticationPayload, False, None) self.failUnless(status, 'testEnclave bridge could not join second bootstrap [%s]' % status) self.bridgeNode = bridgeNode # Store a reference # Send a flooding message to one enclave and verify only one got it. messageNum = random.randint(1,9999999) status = yield TestUtils.sendFlood(bridgeNode,messageNum, allEnclaves[0] ) self.assertTrue(status,"sendFlood failed!") yield TestUtils.wait(3) # Wait for the messages to send # Check message counts status = TestUtils.didReceive(self.allNodeObservers, allEnclaves[0], messageNum, numNodes) self.assertTrue(status,"Nodes in enclave %s didn't all recv message") status = TestUtils.didNotReceive(self.allNodeObservers, allEnclaves[1], messageNum, numNodes) self.assertTrue(status,"Some Nodes in enclave %s did recv message")
def testStartAutoDiscoveryMultipleClients(self): '''Try to start a bootstrap node and the many many clients all at once with autodiscovery on. ''' log.msg("---------------------- BEGIN testStartAutoDiscoveryMultipleClients -------------- ") self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] numNodes = 15 # Create Bootstrap port = 12345 enclaveStr = 'testclave' bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ] (self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList) yield d # Now try and join a bunch of nodes with autodiscovery allDefs = [] # A list of deferreds for counter in range(numNodes): allDefs.append(self.startClientNodeAutoDiscovery(None, self.myIP, port+1+counter, enclaveStr, bootstrapNodeList)) dl = defer.DeferredList(allDefs) yield dl # Wait for all deferred joins to complete # Shut it all down for (clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() if Config.USE_CONNECTION_CACHE: # Try and wait for connection cache to finish disconnects (just in case). yield TestUtils.waitForConnectionCache() else: # Wait a few seconds for network timeouts yield TestUtils.wait(5)
def testDefaultEnclaveAPI(self): '''Create a node which should default to localhost as the enclave name. ''' global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Build the client and network objects enclaveStr1 = None self.bsClient = SampleClient(self.myIP, port, None) self.bsNetwork = classChordNetworkChord(self.bsClient, port, self.myIP) bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests # Join the network log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI") d = defer.Deferred() callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork) self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, True) yield d # Wait for join to succeed # Now check that the node is part of localhost enclave = yield self.bsNetwork.findEnclave(self.myIP, port) self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI did not get localhost as enclave!") # Now create a client node and have it ask the bootstrap node for the enclave log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI") yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, None) # Check that it got enclave localhost enclave = yield self.allNodes[0][1].findEnclave(self.myIP, 12350+1) self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI: client did not get localhost as enclave!") # Now shut everything down for (_clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() # Now wait for the network cache disconnect timeout yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1) defer.returnValue(True)
def test_monoid_identity(self, lst): lst = TestUtils.lst_validate(lst) dictionary_1 = Dictionary() dictionary_1.from_list(lst) dictionary_2 = copy.deepcopy(dictionary_1) dictionary_3 = copy.deepcopy(dictionary_1) # ae = a dictionary_1.mconcat(dictionary_1.mempty()) self.assertEqual(dictionary_1, dictionary_2) # ea = a tmp = dictionary_3.mempty() tmp.mconcat(dictionary_3) self.assertEqual(tmp, dictionary_2)
def testIPFindingAPI(self): '''Create a node and ask the system to figure out it's IP from a bootstrap node. ''' global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Build the client and network objects enclaveStr1 = "AnEnclave" myIP = None # self.myIP self.bsClient = SampleClient(myIP, port, None) self.bsNetwork = classChordNetworkChord(self.bsClient, port, myIP) bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests # Join the network log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI") d = defer.Deferred() callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork) self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, False) yield d # Wait for join to succeed # Now create a client node which will get it's IP from the bootstrap node log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI") yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, enclaveStr1) # Do a bit of verification # Now shut everything down for (_clientAPI, networkAPI) in self.allNodes: rc = yield networkAPI.isConnected("ANY") self.assertTrue(rc != False, "isConnected returned a False value in testIPFindingAPI") # Now shut everything down for (_clientAPI, networkAPI) in self.allNodes: yield networkAPI.disconnect() yield self.bsNetwork.disconnect() # Now wait for the network cache disconnect timeout yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1) defer.returnValue(True)
def testDisconnectedBootstraps(self): '''Create a BS node and some clients. Create another bootstrap node and some clients (so we essentially have two rings). Verify, that the bootstrap nodes autodiscover each other and connect together ''' global startingPort # Create Bootstrap port = 12345 bootstrapNodeLocation = NodeLocation(None, self.myIP, port) bootstrapNodeLocation2 = NodeLocation(None, self.myIP, port+1) self.allNodes = [] self.allMetricsObservers = [] self.allTestObservers = [] # Build the BS node (status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation.ip, bootstrapNodeLocation.port, None, 'theEnclave', True, True) self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI.chordNode)) self.assertTrue(status, 'Could not build bootstrap node') # Build second BS node (status, bsClientAPI2, bsNetworkAPI2) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation2.ip, bootstrapNodeLocation2.port, None, 'theEnclave', True, True) self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI2.chordNode)) self.assertTrue(status, 'Could not build bootstrap node 2') # Build the client node (status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+2, bootstrapNodeLocation, 'theEnclave', False, False) self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI.chordNode)) self.assertTrue(status, 'Could not build client node') # Build the client node (status, clClientAPI2, clNetworkAPI2) = yield TestUtils.startNodeUsingAPI(self.myIP, port+3, bootstrapNodeLocation2, 'theEnclave', False, False) self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI2.chordNode)) self.assertTrue(status, 'Could not build client node') # Wait for flooding to reach all the nodes waiter = ConnectivityCounter() yield waiter.waitForConnectivity(3, clNetworkAPI.chordNode) # Does not count clNode itself. # Now shut everything down yield clNetworkAPI.disconnect() yield clNetworkAPI2.disconnect() yield bsNetworkAPI.disconnect() yield bsNetworkAPI2.disconnect() if Config.USE_CONNECTION_CACHE: yield TestUtils.waitForConnectionCache() else: yield TestUtils.wait(5) defer.returnValue(True)
def testAggregationMessage(self): ''' Send flooding message and ask for aggregation response. ''' print("testAggregationMessage method starting...") counter = 1 try: d = defer.Deferred() messageNum = random.randint(1,9999999) # Reset message count self.observer.resetMessageCount() self.observer.printDebug = True aggMax = numNodes + 2 #aggNumber = 3 for aggNumber in range(1, aggMax+1): #if aggNumber == aggNumber: # Flood from bootstrap to all nodes for i in range(numMessages): #d.addCallback(self.sendFlood, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber) yield self.sendFlood(None, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber) print("sending message %d", counter) counter += 1 d.callback(True) yield d # Wait a few seconds yield TestUtils.wait(5) print("testAggregationMessage check results...") # Now check the results yield self.checkResults(aggMax) except Exception, e: self.anError(e)
def test_scripts(): '''yield list of scripts to test.''' # the current directory current_dir = os.getcwd() # directory location of tests testing_dir = TestUtils.get_tests_directory() # directory location of scripts scripts_dir = os.path.join(os.path.dirname(testing_dir), "CGAT", "scripts") # directories with tests (correspond to script names and # hence end in .py) test_dirs = glob.glob(os.path.join(testing_dir, "*.py")) # the config file config_file = os.path.join(testing_dir, "_test_scripts.yaml") if os.path.exists(config_file): config = yaml.load(open(config_file)) if config is not None: if "restrict" in config and config["restrict"]: values = config["restrict"] if "glob" in values: test_dirs = os.path.join(testing_dir, values["glob"]) if "manifest" in values: # take scripts defined in the MANIFEST.in file test_dirs = [ x for x in open("MANIFEST.in") if x.startswith("include scripts") and x.endswith(".py\n") ] test_dirs = [ re.sub("include\s*scripts/", "tests/", x[:-1]) for x in test_dirs ] if "regex" in values: rx = re.compile(values["regex"]) test_dirs = list(filter(rx.search, test_dirs)) # ignore those which don't exist as tests (files added through MANIFEST.in, # such as version.py, __init__.py, ... test_dirs = [x for x in test_dirs if os.path.exists(x)] # ignore non-directories test_dirs = [x for x in test_dirs if os.path.isdir(x)] test_dirs.sort() # restrict tests run according to chunk parameters starting_test_number = os.getenv('CGAT_TASK_ID', None) test_increment = os.getenv('CGAT_TASK_STEPSIZE', None) try: starting_test_number, test_increment = \ (int(starting_test_number) - 1, int(test_increment)) test_dirs = test_dirs[starting_test_number:starting_test_number + test_increment] except TypeError: pass for test_script in test_dirs: script_name = os.path.basename(test_script) check_main.description = os.path.join(script_name, "def_main") yield (check_main, os.path.abspath(os.path.join(scripts_dir, script_name))) fn = '%s/tests.yaml' % test_script if not os.path.exists(fn): continue script_tests = yaml.load(open(fn)) for test, values in sorted(list(script_tests.items())): check_script.description = os.path.join(script_name, test) if "skip_python" in values: versions = [ x.strip() for x in str(values["skip_python"]).split(",") ] versions = [ x for x in versions if PYTHON_VERSION.startswith(x) ] if len(versions) > 0: continue if "skip_travis" in values and TRAVIS: continue if "skip_jenkins" in values and JENKINS: continue # deal with scripts in subdirectories. These are prefixed # by a "<subdir>_" for example: optic_compare_projects.py # is optic/compare_procjets.py if "_" in script_name: parts = script_name.split("_") if os.path.exists( os.path.join("scripts", parts[0], "_".join(parts[1:]))): script_name = os.path.join(parts[0], "_".join(parts[1:])) yield (check_script, test, os.path.abspath(os.path.join(scripts_dir, script_name)), values.get('stdin', None), values['options'], values['outputs'], values['references'], test_script, current_dir)
def tearDown(self): dirToRemove = "{0}/tmp".format(TestUtils.CreateAndGetTMPDir()) if os.path.exists(dirToRemove): shutil.rmtree(dirToRemove)
def setUp(self): tmpFolder = "{0}/tmp".format(TestUtils.CreateAndGetTMPDir()) if not os.path.exists(tmpFolder): os.makedirs(tmpFolder)
def tearDownClass(): TestUtils.RemoteTMPDir()
def parse_options(argv): """ Parses and checks the command-line options. Returns: A tuple containing the options structure. """ usage = 'Usage: %prog [options]' desc = 'Example: %prog -i flatbufFile -d' parser = optparse.OptionParser(usage=usage, description=desc) parser.add_option('-i', '--flatbuf', dest='input_file', action='store', help='Test flatbuf', metavar='PROTOTEXT_FILE') parser.add_option('-o', '--output', dest='output_dir', action='store', default='./', help='Output directory', metavar='OUTPUT_DIR') parser.add_option('-d', '--debug', action="store_true", dest="log_level", default=False, help='Log Level') parser.add_option('-p', '--port', action="store", dest="port_addr", default=6666, help='Client Port.') parser.add_option('-s', '--doNotShutServer', action="store_false", dest="shut_server", default=False, help='Don\'t Shut Down Server.') parser.add_option('--image', '--img', dest='image_file', action='store', default=None, help='Image file') options, categories = parser.parse_args(argv[1:]) _input = options.input_file outfile_list = list() if options.input_file is None: parser.error('Input is missing') else: options.input_file = os.path.abspath(options.input_file) if not os.path.exists(options.input_file): print("Invalid input directory: ", options.input_file) sys.exit(1) else: options.input_file = tu.get_input_files(options.input_file) if options.image_file is None: print("No img to run") else: options.image_file = os.path.abspath(options.image_file) if not os.path.exists(options.image_file): print("Invalid image location: ", options.image_file) sys.exit(1) else: options.image_file = tu.get_input_files(options.image_file) return (options, categories)
def main(): seqs = cfg.sequences[cfg.hevc_A] + cfg.sequences[cfg.hevc_B] in_names = cfg.class_sequence_names[cfg.hevc_A] + cfg.class_sequence_names[ cfg.hevc_B] ver = 20 shared_param = ("--preset", "ultrafast", "--gop", "0", "--no-tmvp") thread_range = (13, 14, 15, 16) #(4,6,8,10,12,14) owf_range = (1, 2, 3) #(2,4,8,16) outname = "skvz_thread_owf_test_v{}".format(ver) # Set shared param tpg_scal = TU.TestParameterGroup() tpg_scal.add_const_param(version = ver, bin_name = cfg.skvz_ver_bin.format(ver), input_names = in_names, layer_args = shared_param)\ .add_param_set(_thrd = thread_range, _owf = owf_range) tpg_sim = tpg_scal.copy() # Set scalable param round2 = lambda x, base=2: int(base * round(x / base)) strscale = lambda size, scale: "x".join( map(lambda x: str(round2(int(x) * scale)), re.search("_*(\d+)x(\d+)[_.]*", size).group(1, 2))) seq_scale = lambda scale, st: (st[0], strscale(st[1], scale), st[2]) seq_map = lambda scale, seq: r"{}_{}_{}_zerophase_0.9pi.yuv".format( *seq_scale( scale, re.search("(.+\\\\.+)_(\d+x\d+)_(\d+)[_.]", seq).group(1, 2, 3))) bl_seq_map = lambda scale: lambda seq: (seq_map(scale[0], seq[0]), ) scal_seq_map = lambda scale: lambda seq: (seq_map(scale[0], seq[ 0]), ) + seq #TODO: Add scaling to el seq? scal_seqs = tuple(map(scal_seq_map((0.5, 1)), seqs)) tpg_scal.add_const_param(inputs=scal_seqs) tpg_scal.set_param_group_transformer( TU.transformerFactory( test_name=lambda *, _thrd, _owf, **param: "SCAL_THRD{}_OWF{}". format(_thrd, _owf), layer_args=lambda *, layer_args, _thrd, _owf, **param: (layer_args + ("--threads", str(_thrd)) + ("--owf", str(_owf)), layer_args + ("--threads", str(_thrd)) + ("--owf", str(_owf))))) # Set simulcast param tpg_sim.add_const_param(inputs = seqs)\ .add_param_set(_layer=("BL","EL")) tpg_sim.set_param_group_transformer( TU.transformerFactory( test_name=lambda *, _layer, _thrd, _owf, **param: "{}_THRD{}_OWF{}" .format(_layer, _thrd, _owf), layer_args=lambda *, layer_args, _thrd, _owf, **param: (layer_args + ("--threads", str(_thrd)) + ("--owf", str(_owf)), ), inputs=lambda *, inputs, _layer, **param: tuple( map(bl_seq_map((0.5, )), inputs)) if _layer in "BL" else inputs)) #Run tests tests_scal = tpg_scal.to_skvz_test_instance() tests_sim = tpg_sim.to_skvz_test_instance() combi = TU.generate_combi(tpg_sim, combi_cond=TU.combiFactory( _thrd=op.eq, _owf=op.eq, _layer=lambda p1, p2: 0 if p1 == p2 else (-1 if p1 == "BL" else 1))) sim_names = TU.get_combi_names(combi) test_names = TU.get_test_names(tests_scal) + sim_names matrix_summary = TU.make_BDBRMatrix_definition( test_names + TU.get_test_names(tests_sim), write_bdbr=True, write_bits=False, write_psnr=False, layering_func=lambda t: (-1, 1) if "SCAL" not in t else (-1, ), filter_func=lambda t: True if ("BL" in t and "EL" in t) or "SCAL" in t else False) anchor_summary = TU.make_AnchorList_multiAnchor_definition( test_names, global_filter=lambda t: True if "SCAL" in t else False, bdbr_anchor_func=TU.anchorFuncFactory_match_layer(lambda t: tuple( a for a in sim_names if (t.split(sep='_')[1] in a) and (t.split(sep='_')[2] in a))), bdbr_layer_func=TU.layerFuncFactory([ [None, 1], ]), time_anchor_func=TU.anchorFuncFactory_match_layer(lambda t: ( None, ) + tuple((a, l) if l >= 0 else a for a in sim_names if (t.split(sep='_')[1] in a) and (t.split(sep='_')[2] in a) for l in [-1, 1]))) summaries = {sn_BDBRM: matrix_summary, sn_ANCHOR: anchor_summary} runTests(tests_scal + tests_sim, outname, layer_combi=combi, **summaries)
def teardown_module(module): server.terminate() tu.stopNet("scimma-test")
def test_python_len_and_dictionary_size_equality(self, lst): dictionary = Dictionary() lst = TestUtils.lst_validate(lst) dictionary.from_list(lst) self.assertEqual(len(dictionary.to_list()), dictionary.size()[1])
def tearDown(self): TestUtils.RemoteTMPDir()
def setup_module(module): global server tu.startNet("scimma-test") server = tu.ServerContainer(tu.testTag(), opts=["-v", "shared:/root/shared"]) server.start()
def main(): options, categories = parse_options(sys.argv) clientlogfile = options.output_dir + "/DLAClientLogger.log" resultsdir = options.output_dir + "/results/" tu.logger_setup(options.log_level, clientlogfile) dlasocket = ds.dlaSocket() #Set Client Port dlasocket.setPort(options.port_addr) dlasocket.connect(dlasocket.HOST, dlasocket.PORT) dlasocket.setTimeout(dlasocket.getTimeout()) logging.info("DLA Client open at PORT: {0}.".format(dlasocket.getPort())) getWelcomeMessage(dlasocket) test_i = 0 while test_i < len(options.input_file): fbuf_file = options.input_file[test_i] fbuf_size = os.stat(fbuf_file).st_size if options.image_file is None: fbuf_file_name = options.input_file[test_i].split("/")[-1] logging.info("Attempting to read flatbuf: [{0}], " \ "size[{1}].".format(fbuf_file_name, fbuf_size)) readFlatbuf(dlasocket, fbuf_file) logging.info("Attempting to run flatbuf: [{0}], " \ "size[{1}].".format(fbuf_file_name, fbuf_size)) runFlatbuf(dlasocket, fbuf_file) else: img_file = options.image_file[test_i] img_size = os.stat(img_file).st_size fbuf_file_name = options.input_file[test_i].split("/")[-1] logging.info("Attempting to read flatbuf: [{0}], " \ "size[{1}].".format(fbuf_file_name, fbuf_size)) readFlatbuf(dlasocket, fbuf_file) image_file_name = options.image_file[test_i].split("/")[-1] logging.info("Attempting to run image: [{0}], " \ "size[{1}].".format(image_file_name, img_size)) runImage(dlasocket, img_file) numOutputs = getNumOutputs(dlasocket) for ii in range(numOutputs): writeOutput(dlasocket, ii, resultsdir) test_i += 1 #Send ShutDown command to Server if shut_server is True. if options.shut_server: shutDownServer(dlasocket) #Close dlsSocket dlasocket.closeConnection() return 0
def _getOutDir(self): return TestUtils.CreateAndGetTMPDir()
def main(agrv = None): #FLAGS.para_name if FLAGS.Mode == 'Train': #keep_probability = tf.placeholder(tf.float32, name = "keep_prob") #dropout: keep_probability is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img') annotation = tf.placeholder(tf.int32, shape = [None, SP_HEIGHT, SP_WIDTH, 1], name = 'annotation') Testloss = tf.placeholder(tf.float32, name = 'Test_loss') logits, pred = net.PSPUnet(image, is_training) tf.summary.image("image", image, max_outputs = batch_sz) tf.summary.image("groud_truth", tf.cast(annotation, tf.uint8), max_outputs = batch_sz) tf.summary.image("pred_annotation", tf.cast(pred, tf.uint8), max_outputs = batch_sz) #softmax cross entropy loss loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits, labels = tf.squeeze(annotation, axis=[3]), name = 'entropy_loss')) DSC_batch = tf.placeholder(tf.float32, name = 'Dice_coeff') trainable_var = tf.trainable_variables() if FLAGS.Debug: for var in trainable_var: tf.summary.histogram(var.op.name, var) tf.add_to_collection('reg_loss', tf.nn.l2_loss(var)) #BN: update moving_mean&moving_variance update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = Train(loss, trainable_var) # loss_train_op = tf.summary.scalar('training_loss', loss) tf.summary.scalar('training_loss', loss) summary_op = tf.summary.merge_all() loss_test_op = tf.summary.scalar('test_loss',Testloss) DSC_op = tf.summary.scalar('Dice_coefficient', DSC_batch) print("setting up image reader...") image_batch_a, label_batch_a = read_mat.read_and_decord('Train') img_train_batch, label_train_batch = tf.train.shuffle_batch([image_batch_a, label_batch_a], batch_size=batch_sz, capacity=batch_sz*2, min_after_dequeue=batch_sz) print (img_train_batch.shape) print ('setup session...') print("Setting up dataset reader") config = tf.ConfigProto() config.gpu_options.allow_growth=True sess = tf.Session(config=config) print("Setting up Saver...") saver = tf.train.Saver(max_to_keep=50) #filename = 'E:/maning/Cells/mod/model-87500' #saver.restore(sess,filename) summary_writer = tf.summary.FileWriter(FLAGS.logs_dir,sess.graph) sess.run(tf.global_variables_initializer()) threads = tf.train.start_queue_runners(sess) test_i = -1 for itr in range (MAX_ITERATION): #img_batch shape:[batch_size, depth, height, width, chanl], type: ndarray img_batch, l_batch = sess.run([img_train_batch, label_train_batch ]) print (itr, img_batch.shape) feed = {image: img_batch, annotation: l_batch, is_training: True} sess.run(train_op, feed_dict = feed) train_loss_print, summary_str= sess.run([loss,summary_op], feed_dict = feed) print (train_loss_print) summary_writer.add_summary(summary_str, itr) if itr%test_epoch == 0 or itr%5000 == 0: saver.save(sess, './mod/model', global_step=itr) elif itr == (MAX_ITERATION - 1): saver.save(sess, './mod/model', global_step=itr) ############################# test test test test test test test test test test##################################### if (itr!=0 and itr%test_epoch == 0) or itr == (MAX_ITERATION - 1): #overfittest test_i = test_i+1 print("train finish! start test~") test_img_data_a, test_label_data_a = read_mat.read_and_decord('Test') test_img_train_batch, test_label_train_batch = tf.train.batch([test_img_data_a, test_label_data_a],batch_size=1, capacity=1) threads = tf.train.start_queue_runners(sess) Testl = 0.0 for test_itr in range (TEST_RAW): test_img_batch, test_l_batch = sess.run([test_img_train_batch, test_label_train_batch]) test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS) test_lbl_sp = data_pro.image_splite(test_l_batch,COLS,ROWS) pred_sp = {} for col in range(COLS): pred_sp[col]={} for row in range(ROWS): img_test = test_img_sp[col][row] lbl_test = test_lbl_sp[col][row] test_feed = {image: img_test, annotation: lbl_test, is_training: False} test_pred_logits, pred_image, testloss = sess.run([logits, pred, loss], feed_dict = test_feed) pred_sp[col][row] = pred_image Testl = Testl+testloss #pred_image = sess.run(tf.argmax(input=test_pred_logits,axis=3)) pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1) label_batch = np.squeeze(test_l_batch) pred_batch = np.squeeze(pred_all) #label_batch_tp = np.transpose(label_batch, (0, 2, 1)) label_tosave = np.reshape(label_batch, [HEIGHT,WIDTH]) #pred_batch_tp = np.transpose(pred_batch, (0, 2, 1)) pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH]) print("test_itr:",test_itr) # tep = test_pred_annotation[0, 30, :, 0] #np.savetxt('pred30.csv', tep, delimiter=',') #np.savetxt('dice_smi_co.csv',test_dice_coe, delimiter=',') utils.save_imgs(test_itr, label_tosave, pred_tosave, itr) Testl = Testl/TEST_RAW test_summary_str = sess.run(loss_test_op, feed_dict = {Testloss:Testl}) print (test_i,':',Testl) summary_writer.add_summary(test_summary_str, test_i) # Dise similarity coefficient DSC = utils.Accuracy_Measure(itr) DSC_Summary_str = sess.run(DSC_op, feed_dict = {DSC_batch:DSC}) print (test_i,':',DSC) summary_writer.add_summary(DSC_Summary_str, test_i) elif FLAGS.Mode == 'Visualize': start = time.time() is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img') logits, pred = net.PSPUnet(image, is_training) print ('setup session...') print("Setting up dataset reader") config = tf.ConfigProto() config.gpu_options.allow_growth=True sess = tf.Session(config=config) print("Setting up Saver...") saver = tf.train.Saver(max_to_keep=50) sess.run(tf.global_variables_initializer()) saver.restore(sess,modelname) img_folder = img_path test_itr=0 for imgname in os.listdir(img_folder): test_itr = test_itr+1 print('path is :', img_folder+imgname) test_img_batch = testutils.read_img(img_folder+imgname) test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS) pred_sp = {} logits_sp = {} for col in range(COLS): pred_sp[col]={} logits_sp[col] = {} for row in range(ROWS): img_test = test_img_sp[col][row] test_feed = {image: img_test, is_training: False} test_pred_logits, pred_image = sess.run([logits, pred], feed_dict = test_feed) logits_sp[col][row] = test_pred_logits pred_sp[col][row] = pred_image pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1) pred_batch = np.squeeze(pred_all) print("test_itr:",test_itr) testutils.saveImage(img_folder+imgname, pred_batch, imgname, test_save_path) end = time.time() elapse = end - start print('elapse time is :', elapse) print ('finished!')