def main(agrv=None):
start = time.time()
is_training = tf.placeholder(tf.bool, name='is_train') #BN: istraining
image = tf.placeholder(tf.float32,
shape=[None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL],
name='input_img')
# logits, pred = net.u_net(image, is_training)
logits, pred = net.PSPUnet(image, is_training)
print('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
sess.run(tf.global_variables_initializer())
saver.restore(sess, modelname)
img_folder = img_path
test_itr = 0
for imgname in os.listdir(img_folder):
test_itr = test_itr + 1
print('path is :', img_folder + imgname)
test_img_batch = testutils.read_img(img_folder + imgname)
test_img_sp = data_pro.image_splite(test_img_batch, COLS, ROWS)
pred_sp = {}
logits_sp = {}
for col in range(COLS):
pred_sp[col] = {}
logits_sp[col] = {}
for row in range(ROWS):
img_test = test_img_sp[col][row]
test_feed = {image: img_test, is_training: False}
test_pred_logits, pred_image = sess.run([logits, pred],
feed_dict=test_feed)
logits_sp[col][row] = test_pred_logits
pred_sp[col][row] = pred_image
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
pred_batch = np.squeeze(pred_all)
print("test_itr:", test_itr)
testutils.saveImage(img_folder + imgname, pred_batch, imgname,
test_save_path)
# pred_batch = np.squeeze(pred_all)
# pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH])
# print("test_itr:",test_itr)
# utils.save_Test(imgname[:-4], pred_tosave, test_save_path)
end = time.time()
elapse = end - start
print('elapse time is :', elapse)
def testNodeStartup(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, self.normalNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12346, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
# Are they connected together?
status = yield self.doQuickSendTest()
self.assertTrue(status, 'doQuickSendTest')
# Stop the nodes
yield self.normalNode.leave()
yield self.bsNode.leave()
# Wait for the connections to really all close
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
else:
yield TestUtils.wait(3)
defer.returnValue(True)
def testNodeStartup(self):
port = 12345
enclave = 'localhost'
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
# Start a bootstrap node
(status, bsNode, _) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, 'Could not build bootstrap node')
# Start a client node
(status, node, _) = yield TestUtils.startupClientNode(self.myIP, 12346, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
#yield TestUtils.wait(3)
# Are they connected together?
status = yield self.doConnectionTest(node)
self.assertTrue(status, 'doConnectionTest')
# Stop the nodes
yield bsNode.leave()
yield node.leave()
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 3)
defer.returnValue(True)
def getAggResponse(self):
'''Send a query and get an agg response from all the nodes.'''
bsNode = self.bsNetwork.chordNode
for aggNumber in range(1, numNodes + 5):
# Reset the message counter
obs = self.getObserver(bsNode, self.allTestObservers)
obs.resetMessageCount()
# Send the query asking for a response
messageNum = aggNumber # Just to keep it incrementing
yield TestUtils.sendFlood(bsNode,messageNum,'localhost', data="SEND_AGG_RESPONSE:%d" % aggNumber)
# Now check how many messages the bootstrap node got in return
for _ in range(10): # 10 seconds
recvCount = obs.getMessageCount()
if recvCount >= numNodes+1:
break
else:
yield TestUtils.wait(1)
self.failUnless(recvCount == numNodes+1, "getAggResponse didn't get the correct number of messages back.Expected[%d] Got[%d]" % (numNodes+1, recvCount))
defer.returnValue(True)
def testSerialP2PSending(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
log.msg("Building nodes...")
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Do the real test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two
yield TestUtils.wait(3+Config.CONNECTION_CACHE_DELAY)
defer.returnValue(True)
def test_iter_and_next(self):
lst = [('age', 23), ('score', 95), ('color', 'blue')]
dictionary = Dictionary()
dictionary.from_list(lst)
tmp_1 = []
for key, value in dictionary:
tmp_1.append((key, value))
sorted_tmp = TestUtils.sort(tmp_1)
t = dictionary.to_list()
self.assertEqual(t, sorted_tmp)
# Test for two iterators on one data structure working in parallel.
tmp_2 = []
tmp_3 = [('age', 'age'), ('age', 'score'), ('age', 'color'),
('score', 'age'), ('score', 'score'), ('score', 'color'),
('color', 'age'), ('color', 'score'), ('color', 'color')]
for iterator_1 in dictionary:
for iterator_2 in dictionary:
tmp_2.append((iterator_1[0], iterator_2[0]))
tmp_2 = TestUtils.sort(tmp_2)
tmp_3 = TestUtils.sort(tmp_3)
self.assertEqual(tmp_2, tmp_3)
it = iter(Dictionary())
self.assertRaises(StopIteration, lambda: next(it))
def test_add(self):
dictionary = Dictionary()
lst = []
dictionary.add("score", 89)
lst.append(("score", 89))
self.assertEqual(dictionary.to_list(), [("score", 89)])
dictionary.add("score", 78)
lst.append(("score", 78))
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
dictionary.add("gender", "male")
lst.append(("gender", "male"))
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
dictionary2 = Dictionary()
temp1 = (23, 34)
temp2 = (23, 34)
dictionary2.add(temp1, 'test_value_1')
dictionary2.add(temp2, 'value2')
lst1 = [(temp2, 'value2'), (temp1, 'test_value_1')]
self.assertEqual(dictionary2.to_list(), TestUtils.sort(lst1))
# Exception test
# 1. Add repeated key-value.
dictionary.add("score", 78)
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
# 2. Add invalid key
self.assertRaises(Exception, dictionary.add, {
'key1': 2,
"key2": 3
}, "dict_key_test")
# 3. Add None value
self.assertRaises(Exception, dictionary.add, "score", None)
def testSerialFlooding(self):
# Start a bootstrap node
(status, self.bsNode, _observer) = yield TestUtils.startupBootstrapNode(self.myIP, 12345, 'localhost')
self.assertTrue(status, 'Could not build bootstrap node')
self.allNodes.append(self.bsNode)
self.bsNode.addMessageObserver(self.messageReceived)
# Start client nodes
for i in range(numNodes):
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, 12346+i, 'localhost', self.bsNode.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
self.allNodes.append(node)
observer = MyMessageObserver()
node.addMessageObserver(observer.messageReceived)
observer.node = node
self.allObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode) # Does not count bsNode itself.
# Now do the stress test
status = yield self.doStressTest()
# Now close it all down!
yield self.allLeave()
# Wait a second or two for network timeouts
yield TestUtils.wait(9)
defer.returnValue(True)
def test_associativity(self, lst_1, lst_2, lst_3):
lst_1 = TestUtils.lst_validate(lst_1)
lst_2 = TestUtils.lst_validate(lst_2)
lst_3 = TestUtils.lst_validate(lst_3)
dictionary_1 = Dictionary()
dictionary_1.from_list(lst_1)
dictionary_2 = Dictionary()
dictionary_2.from_list(lst_2)
dictionary_3 = Dictionary()
dictionary_3.from_list(lst_3)
# It equals to "(ab)c".
dictionary_1.mconcat(dictionary_2)
dictionary_1.mconcat(dictionary_3)
temp1 = dictionary_1
dictionary_1 = Dictionary()
dictionary_1.from_list(lst_1)
dictionary_2 = Dictionary()
dictionary_2.from_list(lst_2)
# It equals to "a(bc)".
dictionary_2.mconcat(dictionary_3)
dictionary_1.mconcat(dictionary_2)
temp2 = dictionary_1
assert id(temp2) != id(temp1)
# To determine if "(ab)c = a(bc)" holds.
self.assertEqual(temp1, temp2)
def sendP2PMsg(self, src, dst, allNodeObservers):
'''Send a P2P message from src to dst and verify it got there.
Also check that no other nodes got it.
'''
goodCounter = 0
for _ in range(numMessages):
messageNum = random.randint(1,9999999)
# Send the message
status = yield TestUtils.sendP2P(src, dst, messageNum )
self.assertTrue(status,"sendP2PMsg failed! [%s] [%s] to [%s]" % (status, src, dst))
#yield TestUtils.wait(1) # Wait for the messages to send
# Check receipt (both that one did and others did not)
status = TestUtils.didNodeReceive(allNodeObservers,dst, messageNum)
self.assertTrue(status,"Expected node did not receive the message!")
status = TestUtils.didNotReceive(allNodeObservers, 'ALL', messageNum, len(allNodeObservers) - 1)
self.assertTrue(status,"Extra nodes received message!")
goodCounter += 1
self.assertTrue(goodCounter == numMessages, "sendP2P message did not finish all messages [%d][%d]" % (goodCounter, numMessages))
defer.returnValue(True)
def test_copy_force(self):
testFileName = "test_file.txt"
data = {
"target":
"{0}/{1}".format(TestUtils.CreateAndGetTMPDir(), testFileName),
"to":
"{0}/tmp".format(TestUtils.CreateAndGetTMPDir()),
"force":
True
}
with open(data["target"], 'w+') as fileObj:
fileObj.write("1")
resFile = "{0}/{1}".format(data["to"], testFileName)
with open(resFile, 'w+') as fileObj:
fileObj.write("2")
step = CopyFile()
step.serialize(data)
self.assertTrue(step.run())
self.assertTrue(os.path.exists(resFile))
with open(resFile, 'r') as fileObj:
data = fileObj.read()
self.assertEqual(data, "2")
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
enclave = "theEnc"
# Start a bootstrap node
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, enclave)
self.assertTrue(status, "Could not build bootstrap node")
self.allBootstrapObservers.append(observer)
# Add X nodes to each enclave
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(
self.myIP, startingPort, enclave, bootstrapNodeLocation
)
self.assertTrue(status, "Could not startupClientNode")
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes + 1, self.bsNode) # Does not bsNode .
defer.returnValue(True)
def createOneWrongArray(size, count):
arrays[size] = []
for i in range(count):
random.seed(i)
array = list(range(size))
if size > 1:
a = b = -1
while a == b:
a = random.randint(0, size - 1)
b = random.randint(0, size - 1)
TestUtils.swap(array, a, b)
arrays[size].append(array)
def test_map_my(self):
lst = [('score', [98, {99, 100}]), ('age', 23), ('length', 50)]
dictionary = Dictionary()
dictionary.from_list(lst)
dictionary.map_my(lambda x: x + 1)
result = [('age', 24), ('length', 51), ('score', [99, [100, 101]])]
self.assertEqual(dictionary.to_list(), TestUtils.sort(result))
def testStartAutoDiscoveryClientTimeout(self):
'''Try to start a bootstrap node with autodiscovery on.
This should show a failure notice.
'''
log.msg("---------------------- BEGIN testStartAutoDiscoveryClientTimeout -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
# Now try and join with autodiscovery
d = defer.Deferred()
clientAPI = SampleClient(self.myIP, port, None)
networkAPI = classChordNetworkChord(clientAPI, port, self.myIP)
nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
callFunc = lambda result, payload: self.shouldFailCallback(result, payload, d)
networkAPI.start(callFunc, nodeID, enclaveStr, "authenticate:succeed", None, False, True)
yield d
yield TestUtils.waitForConnectionCache()
def getTestSuite(select="unit"):
"""
Get test suite
select is one of the following:
"unit" return suite of unit tests only
"component" return suite of unit and component tests
"all" return suite of unit, component and integration tests
"pending" return suite of pending tests
name a single named test to be run
"""
testdict = {
"unit":
[ "testUnits"
, "testNull"
],
"component":
[ "testComponents"
],
"integration":
[ "testIntegration"
],
"pending":
[ "testPending"
]
}
return TestUtils.getTestSuite(SparqlQueryTestCase, testdict, select=select)
def testMultipleBSNodesAutoDiscoveryClient(self):
'''Try to start multiple bootstrap nodes then a client and make sure it connects to one of them.
'''
log.msg("---------------------- BEGIN testMultipleBSNodesAutoDiscoveryClient -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bootstrapNodeList = [ NodeLocation(None, self.myIP, port), NodeLocation(None, self.myIP, port+1)]
(self.bsClient1, self.bsNetwork1, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
(self.bsClient2, self.bsNetwork2, d) = self.startBootstrapNode(enclaveStr, self.myIP, port+1, "authenticate:succeed", None)
yield d
# Now try and join with autodiscovery
yield self.startClientNodeAutoDiscovery(None, self.myIP, port+2, enclaveStr, bootstrapNodeList)
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork1.disconnect()
yield self.bsNetwork2.disconnect()
yield TestUtils.waitForConnectionCache()
def getTestSuite(select="unit"):
"""
Get test suite
select is one of the following:
"unit" return suite of unit tests only
"component" return suite of unit and component tests
"all" return suite of unit, component and integration tests
"pending" return suite of pending tests
name a single named test to be run
"""
testdict = {
"unit":
[
"oai_listIdentifiers"
],
"component":
[ "testComponents"
],
"integration":
[ "testIntegration"
],
"pending":
[ "testPending"
]
}
return TestUtils.getTestSuite(TestOaiClient, testdict, select=select)
def waitForAllReceived(self):
'''Wait until all messages have been received by all the nodes.'''
numTries = 10
expectedNumMsg = numNodes*numMessages
for _ in range(numTries):
completedObservers = 0
incompleteObservers = 0
# Wait a sec
yield TestUtils.wait(1)
# Count them
for obs in self.allObservers:
numRx = obs.getNumReceived()
if numRx == expectedNumMsg:
completedObservers += 1
elif numRx > expectedNumMsg:
obs.printMessages()
raise Exception("Programming error... received more messages than possible! Got[%d] Expected[%d]" % (numRx,expectedNumMsg ))
else:
incompleteObservers += 1
print("waitForAllReceived: Complete:%d Incomplete:%d" % (completedObservers, incompleteObservers))
if incompleteObservers == 0:
defer.returnValue(True)
defer.returnValue(False)
def filter_files(files):
'''filter list of files according to filters set in
configuration file tests/_test_commandline.yml'''
# directory location of tests
testing_dir = TestUtils.get_tests_directory()
# the config file
config_file = os.path.join(testing_dir, "_test_commandline.yml")
if os.path.exists(config_file):
config = yaml.load(open(config_file))
if config is not None:
if "restrict" in config and config["restrict"]:
values = config["restrict"]
if "manifest" in values:
# take scripts defined in the MANIFEST.in file
scriptdirs = [
x for x in open("MANIFEST.in")
if x.startswith("include CGAT/tools")
and x.endswith(".py\n")
]
take = set(
[re.sub("include\s*", "", x[:-1]) for x in scriptdirs])
files = [x for x in files if x in take]
if "regex" in values:
rx = re.compile(values["regex"])
files = filter(rx.search, files)
return files
def testStartAutoDiscoveryClient(self):
'''Try to start a bootstrap node with autodiscovery on.
'''
log.msg("---------------------- BEGIN testStartAutoDiscoveryClient -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bsNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeList = [ bsNodeLocation ]
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now try and join with autodiscovery
yield self.startClientNodeAutoDiscovery(None, self.myIP, port+1, enclaveStr, bootstrapNodeList)
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
def waitForConnectivity(self, numToWaitFor, chordNode):
'''Wait till we can connect to all numNodes'''
self.node = chordNode
self.testCounter = 1
self.connectedNodeList = []
# Need to see the messages
self.node.addMessageObserver(self.messageReceived)
yield self.getNumConnected()
while len(self.connectedNodeList) < numToWaitFor:
log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter")
self.testCounter += 1
yield self.getNumConnected()
yield TestUtils.wait(5) # Wait for messages to go around
log.msg("DEBUG: waiting for %d nodes. Got %d" % (numToWaitFor, len(self.connectedNodeList)), system="ConnectivityCounter")
# for n in self.connectedNodeList:
# print("DEBUG: Node:%s" % n)
# Don't care anymore
self.node.removeMessageObserver(self.messageReceived)
def test_copy_invalid_file(self):
testFileName = "test_file.txt"
data = {
"target":
"{0}/{1}".format(TestUtils.CreateAndGetTMPDir(), testFileName),
"to":
"{0}/tmp".format(TestUtils.CreateAndGetTMPDir()),
"force":
False
}
self.assertFalse(os.path.exists(data["target"]))
resFile = "{0}/{1}".format(data["to"], testFileName)
step = CopyFile()
step.serialize(data)
self.assertFalse(step.run())
self.assertFalse(os.path.exists(resFile))
def test_from_list(self):
lst = [('name', 'Nick'), ('age', 23), ('gender', 'male'),
('others', [10, 100])]
lst_2 = [('name', 'Nick'), ('others', [10, 100, 200])]
dictionary = Dictionary()
dictionary.from_list(lst)
self.assertEqual(dictionary.get_by_key('gender'), 'male')
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
# Existing dictionary object calls from_lst().
dictionary.from_list(lst_2)
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst_2))
# Exception test
# There are invalid key or value in the list.
lst_3 = [('name', None), ('others', [10, 100, 200]), (None, '10')]
self.assertRaises(Exception, dictionary.from_list, lst_3)
def doTest():
TestDataCube.doTest()
TestShape.doTest()
TestSlice.doTest()
TestTimer.doTest()
TestUtils.doTest()
TestDtypeInfo.doTest()
TestNdArray.doTest()
TestMethods.doTest()
TestCoordinates.doTest()
TestConstants.doTest()
TestLinalg.doTest()
TestRandom.doTest()
TestRotations.doTest()
TestFilters.doTest()
TestPolynomial.doTest()
TestFFT.doTest()
TestImageProcessing.doTest()
def waitForMessageCount(self, obs, numMessagesToWaitFor):
for _ in range(10):
if obs.getPingbackCount() >= numMessagesToWaitFor:
defer.returnValue(True)
else:
yield TestUtils.wait(1)
defer.returnValue(False)
def buildNetwork(self):
global startingPort
# Create two Bootstrap nodes
port = 12345
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
(status, self.bsNode1, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Start another bootstrap node
(status, self.bsNode2, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port+1, allEnclaves[1])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Add some nodes to each bootstrap node
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[0], self.bsNode1.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
for _ in range(numNodes):
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, allEnclaves[1], self.bsNode2.nodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes, self.bsNode1) # Does not count bsNode itself.
yield waiter.waitForConnectivity(numNodes, self.bsNode2) # Does not count bsNode itself.
defer.returnValue(True)
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allNetworkAPIs = []
self.allClientAPIs = []
# Start a bootstrap node
log.msg("building BS node...")
(status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port, None, 'theEnclave', False, True)
self.assertTrue(status, 'Could not build bootstrap node')
node = networkAPI.chordNode # This is very highly coupled -- bad
MetricsMessageObserver(node)
self.observer = TestMessageObserver(node, networkAPI)
self.bsNode = node
log.msg("building client nodes...")
# Add X nodes to each enclave
for _ in range(numNodes):
# Add a node to the enclave
(status, clientAPI, networkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, startingPort, bootstrapNodeLocation, 'theEnclave', False, False)
self.assertTrue(status, 'Could not startupClientNode')
node = networkAPI.chordNode # This is very highly coupled -- bad
MetricsMessageObserver(node)
observer = TestMessageObserver(node, networkAPI)
startingPort += 1
self.allNetworkAPIs.append(networkAPI)
self.allNodeObservers.append(observer)
self.allClientAPIs.append(clientAPI)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(numNodes+1, self.bsNode) # Does count bsNode itself.
defer.returnValue(True)
def doTearDown(self):
'''Tear down the network created during setup.'''
log.msg("tearDown begins...")
# Stop everything
for networkAPI in self.allNetworkAPIs:
yield networkAPI.disconnect()
yield self.bsNode.leave()
# Wait for all network timeouts to finish
#self.flushLoggedErrors()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 2)
else:
yield TestUtils.defWait(3)
defer.returnValue(True)
def test_set_value(self):
dictionary = Dictionary()
dictionary.add("age", 23)
dictionary.set_value("age", 100)
self.assertEqual(dictionary.get_by_key("age"), 100)
# Set a value to a key which not exist.
dictionary.set_value("color", "blue")
self.assertEqual(dictionary.to_list(),
TestUtils.sort([("age", 100), ("color", "blue")]))
def testAggregationMessageAPI(self):
'''
By the time this function is called, we should have a full network built and connected.
Tests to perform now:
1. Send P2P messages from BS Node to all nodes
2. Send P2P messages from all Nodes to BSNode
3. Send flooding from BSNode to all
4. Send query and return aggregation responses of different levels
Finally, do cleanup.
'''
try:
# Build a network
yield self.buildNetwork()
# 1. Send P2P messages from BS Node to all nodes
yield self.p2pFromBStoAll()
# 2. Send P2P messages from all Nodes to BSNode
yield self.p2pFromAlltoBS()
# 3. Send flooding from BSNode to all
yield self.floodFromBStoAll()
# 4. Send query and return aggregation responses of different levels
yield self.getAggResponse()
# Do the cleanup
yield TestUtils.wait(5)
# Shut it all down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 5)
except Exception, e:
log.err(e, "An error occurred in testAggregationMessageAPI")
def test_remove_key(self):
dictionary = Dictionary()
dictionary.add("name", "Nick")
dictionary.add("age", 23)
dictionary.add("gender", "male")
dictionary.add("score", 10)
dictionary.add("score", 100)
dictionary.remove_key("gender")
lst = [("age", 23), ("name", "Nick"), ("score", 10), ("score", 100)]
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
dictionary.remove_key("score")
lst = [("age", 23), ("name", "Nick")]
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
# Exception test
# 1. Remove elements that do not exist.
self.assertRaises(Exception, dictionary.remove_key, 23)
# 2. Remove None key.
self.assertRaises(Exception, dictionary.remove_key, None)
def test_to_list(self):
dictionary = Dictionary()
self.assertEqual(dictionary.to_list(), [])
dictionary.add("name", "Nick")
dictionary.add("age", 23)
dictionary.add("gender", "male")
dictionary.add("others", 10)
dictionary.add("others", 100)
lst = [("age", 23), ("gender", "male"), ("name", "Nick"),
("others", 10), ("others", 100)]
self.assertEqual(dictionary.to_list(), TestUtils.sort(lst))
def __init__(self,
tag,
name="scimma-tst-srv",
net="scimma-test",
opts=[],
dbg=False):
super(ServerContainer, self).__init__(name=name)
self.tag = tag
self.cimage = tu.clientImage(tag)
self.simage = tu.serverImage(tag)
self.name = name
self.options = opts
self.network = net
self.q = multiprocessing.Queue()
self.cmd = "docker"
self.args = [
"run", "--detach=true", "--rm=true",
"--network=%s" % net, "-v", "shared:/root/shared",
"--name=%s" % name
] + opts + [self.simage]
def doTest():
TestConstants.doTest()
TestCoordinates.doTest()
TestDataCube.doTest()
TestDtypeInfo.doTest()
TestFilters.doTest()
TestFunctions.doTest()
TestImageProcessing.doTest()
TestIntegrate.doTest()
TestLinalg.doTest()
TestNdArray.doTest()
TestPolynomial.doTest()
TestRandom.doTest()
TestRotations.doTest()
TestRoots.doTest()
TestShape.doTest()
TestSlice.doTest()
TestSpecial.doTest()
TestTimer.doTest()
TestUtils.doTest()
TestVector.doTest()
def buildNetwork(self):
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodeObservers = []
self.allBootstrapObservers = []
self.allNodes = []
# Start a bootstrap node
log.msg("---> Startup Bootstrap Node", system="EnclaveP2P Test")
(status, self.bsNode, observer) = yield TestUtils.startupBootstrapNode(self.myIP, port, allEnclaves[0])
self.assertTrue(status, 'Could not build bootstrap node')
self.allBootstrapObservers.append(observer)
# Join another enclave
enableAutoDiscovery = True
bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ]
status = yield self.bsNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, None, isBootstrapNode=True, enclaveStr=allEnclaves[1])
# Add X nodes to each enclave
for enclave in allEnclaves:
for _ in range(numNodes):
log.msg("---> Add node to enclave", system="EnclaveP2P Test")
# Add a node to the enclave
(status, node, observer) = yield TestUtils.startupClientNode(self.myIP, startingPort, enclave, bootstrapNodeLocation)
self.assertTrue(status, 'Could not startupClientNode')
startingPort += 1
self.allNodes.append(node)
self.allNodeObservers.append(observer)
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
numberOfNodes = (len(allEnclaves) * numNodes)
yield waiter.waitForConnectivity(numberOfNodes+1, self.bsNode) # Does not count bsNode itself.
defer.returnValue(True)
def testStartAutoDiscoverySameNodeError(self):
'''Try to start a bootstrap node and then two clients on the same port with autodiscovery on.
Should log an error when the second node attempts to join.
'''
log.msg("---------------------- BEGIN testStartAutoDiscoverySameNodeError -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bsNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeList = [ bsNodeLocation ]
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now startup one client which will succeed
d2 = defer.Deferred()
port = port + 1
clientAPI = SampleClient(self.myIP, port, None)
networkAPI = classChordNetworkChord(clientAPI, port, self.myIP)
nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
callFunc = lambda result, payload: self.shouldSucceedCallback(result, payload, d2, networkAPI)
networkAPI.start(callFunc, nodeID, enclaveStr, "authenticate:succeed", bootstrapNodeList, False, True)
yield d2
# Now startup one which should fail due to duplicate port
d3 = defer.Deferred()
clientAPI2 = SampleClient(self.myIP, port, None)
networkAPI2 = classChordNetworkChord(clientAPI2, port, self.myIP)
nodeID = networkAPI.generateNodeID(str(port), enclaveStr) # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
callFunc2 = lambda result, payload: self.shouldFailureCallback(result, payload, d3)
networkAPI2.start(callFunc2, nodeID, enclaveStr, "authenticate:succeed",bootstrapNodeList, False, True)
yield d3
yield networkAPI.disconnect()
#yield networkAPI2.disconnect()
yield self.bsNetwork.disconnect()
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
def testNodeAuth(self):
log.msg("---> Starting BootstrapNode")
yield self.startBootStrapNode()
log.msg("---> Creating Good Client")
yield self.startClientNode(self.myIP, 12350, self.bootstrapNodeLocation, self.enclaveStr)
log.msg("---> Creating Bad Client")
yield self.startBadClientNode(self.myIP, 12351, self.bootstrapNodeLocation, self.enclaveStr)
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(2)
defer.returnValue(True)
def testAutoDiscoveryRebootstrap(self):
'''Try to start a bootstrap node and client with autodiscovery on.
Then start another bootstrap node.
Then kill first bootstrap node and have client try to re-bootstrap to
new bootstrap node.
'''
log.msg("---------------------- BEGIN testAutoDiscoveryRebootstrap -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bsNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeList = [ bsNodeLocation ]
log.msg("Starting bootstrap... \n\n")
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now try and join with autodiscovery
yield TestUtils.wait(1)
log.msg("Starting client... \n\n")
yield self.startClientNodeAutoDiscovery(None, self.myIP, port+1, enclaveStr, bootstrapNodeList)
# Now start a second bootstrap node
#NOT DONE YET
log.msg("Shutting down client... \n\n")
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
def sendFailingP2PMsg(self, src, dst):
'''Send a P2P message from src to dst that should fail to get there
because they are in different enclaves.
Also check that no other nodes got it.
'''
for _ in range(numMessages):
messageNum = random.randint(1,9999999)
# Send the message
status = yield TestUtils.sendP2P(src, dst, messageNum )
self.assertFalse(status,"sendP2PMsg succeeded but should have failed! [%s] to [%s]" % (src, dst))
defer.returnValue(True)
def testEnclaveBridge(self):
'''Create a node that can bridge between the enclaves
Send a flooding message from it to 1 enclave and verify only nodes in that one got it
'''
authenticationPayload = "This is a test"
# Build the bridge node
(status, bridgeNode, observer) = yield TestUtils.startupClientNode(self.myIP, 12500, allEnclaves[0], self.bsNode1.nodeLocation)
self.failUnless(status, 'testEnclave bridge could not join first bootstrap [%s]' % status)
# Join the other enclave also.
enableAutoDiscovery = True
bootstrapNodeList = [ self.bsNode2.nodeLocation ]
status = yield bridgeNode.joinEnclave(bootstrapNodeList, enableAutoDiscovery, authenticationPayload, False, None)
self.failUnless(status, 'testEnclave bridge could not join second bootstrap [%s]' % status)
self.bridgeNode = bridgeNode # Store a reference
# Send a flooding message to one enclave and verify only one got it.
messageNum = random.randint(1,9999999)
status = yield TestUtils.sendFlood(bridgeNode,messageNum, allEnclaves[0] )
self.assertTrue(status,"sendFlood failed!")
yield TestUtils.wait(3) # Wait for the messages to send
# Check message counts
status = TestUtils.didReceive(self.allNodeObservers, allEnclaves[0], messageNum, numNodes)
self.assertTrue(status,"Nodes in enclave %s didn't all recv message")
status = TestUtils.didNotReceive(self.allNodeObservers, allEnclaves[1], messageNum, numNodes)
self.assertTrue(status,"Some Nodes in enclave %s did recv message")
def testStartAutoDiscoveryMultipleClients(self):
'''Try to start a bootstrap node and the many many clients all at once with autodiscovery on. '''
log.msg("---------------------- BEGIN testStartAutoDiscoveryMultipleClients -------------- ")
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
numNodes = 15
# Create Bootstrap
port = 12345
enclaveStr = 'testclave'
bootstrapNodeList = [ NodeLocation(None, self.myIP, port) ]
(self.bsClient, self.bsNetwork, d) = self.startBootstrapNode(enclaveStr, self.myIP, port, "authenticate:succeed", bootstrapNodeList)
yield d
# Now try and join a bunch of nodes with autodiscovery
allDefs = [] # A list of deferreds
for counter in range(numNodes):
allDefs.append(self.startClientNodeAutoDiscovery(None, self.myIP, port+1+counter, enclaveStr, bootstrapNodeList))
dl = defer.DeferredList(allDefs)
yield dl # Wait for all deferred joins to complete
# Shut it all down
for (clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
if Config.USE_CONNECTION_CACHE:
# Try and wait for connection cache to finish disconnects (just in case).
yield TestUtils.waitForConnectionCache()
else:
# Wait a few seconds for network timeouts
yield TestUtils.wait(5)
def testDefaultEnclaveAPI(self):
'''Create a node which should default to localhost as the enclave name. '''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the client and network objects
enclaveStr1 = None
self.bsClient = SampleClient(self.myIP, port, None)
self.bsNetwork = classChordNetworkChord(self.bsClient, port, self.myIP)
bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI")
d = defer.Deferred()
callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, True)
yield d # Wait for join to succeed
# Now check that the node is part of localhost
enclave = yield self.bsNetwork.findEnclave(self.myIP, port)
self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI did not get localhost as enclave!")
# Now create a client node and have it ask the bootstrap node for the enclave
log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI")
yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, None)
# Check that it got enclave localhost
enclave = yield self.allNodes[0][1].findEnclave(self.myIP, 12350+1)
self.assertEqual(enclave, "localhost", "testDefaultEnclaveAPI: client did not get localhost as enclave!")
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1)
defer.returnValue(True)
def test_monoid_identity(self, lst):
lst = TestUtils.lst_validate(lst)
dictionary_1 = Dictionary()
dictionary_1.from_list(lst)
dictionary_2 = copy.deepcopy(dictionary_1)
dictionary_3 = copy.deepcopy(dictionary_1)
# ae = a
dictionary_1.mconcat(dictionary_1.mempty())
self.assertEqual(dictionary_1, dictionary_2)
# ea = a
tmp = dictionary_3.mempty()
tmp.mconcat(dictionary_3)
self.assertEqual(tmp, dictionary_2)
def testIPFindingAPI(self):
'''Create a node and ask the system to figure out it's IP from a bootstrap node. '''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the client and network objects
enclaveStr1 = "AnEnclave"
myIP = None # self.myIP
self.bsClient = SampleClient(myIP, port, None)
self.bsNetwork = classChordNetworkChord(self.bsClient, port, myIP)
bsID = self.bsNetwork.generateNodeID(str(port), "WHO CARES") # Get the ID with the bits on it we need. Use "port" because it'll be uniq for tests
# Join the network
log.msg("---- Bootstrap Join 1 ---- ", system="testDefaultEnclaveAPI")
d = defer.Deferred()
callFunc = lambda x, payload: self.shouldSucceedCallback(x, payload, d, self.bsNetwork)
self.bsNetwork.start(callFunc, bsID, enclaveStr1, "authenticate:succeed", None, True, False)
yield d # Wait for join to succeed
# Now create a client node which will get it's IP from the bootstrap node
log.msg("---- Start Client 1 ---- ", system="testIPFindingAPI")
yield self.startClientNode(None, None, 12350+1, bootstrapNodeLocation, enclaveStr1)
# Do a bit of verification
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
rc = yield networkAPI.isConnected("ANY")
self.assertTrue(rc != False, "isConnected returned a False value in testIPFindingAPI")
# Now shut everything down
for (_clientAPI, networkAPI) in self.allNodes:
yield networkAPI.disconnect()
yield self.bsNetwork.disconnect()
# Now wait for the network cache disconnect timeout
yield TestUtils.wait(Config.CONNECTION_CACHE_DELAY + 1)
defer.returnValue(True)
def testDisconnectedBootstraps(self):
'''Create a BS node and some clients. Create another bootstrap node and some clients (so we essentially have two rings).
Verify, that the bootstrap nodes autodiscover each other and connect together
'''
global startingPort
# Create Bootstrap
port = 12345
bootstrapNodeLocation = NodeLocation(None, self.myIP, port)
bootstrapNodeLocation2 = NodeLocation(None, self.myIP, port+1)
self.allNodes = []
self.allMetricsObservers = []
self.allTestObservers = []
# Build the BS node
(status, bsClientAPI, bsNetworkAPI) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation.ip, bootstrapNodeLocation.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build bootstrap node')
# Build second BS node
(status, bsClientAPI2, bsNetworkAPI2) = yield TestUtils.startNodeUsingAPI(bootstrapNodeLocation2.ip, bootstrapNodeLocation2.port, None, 'theEnclave', True, True)
self.allMetricsObservers.append(MetricsMessageObserver(bsNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build bootstrap node 2')
# Build the client node
(status, clClientAPI, clNetworkAPI) = yield TestUtils.startNodeUsingAPI(self.myIP, port+2, bootstrapNodeLocation, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI.chordNode))
self.assertTrue(status, 'Could not build client node')
# Build the client node
(status, clClientAPI2, clNetworkAPI2) = yield TestUtils.startNodeUsingAPI(self.myIP, port+3, bootstrapNodeLocation2, 'theEnclave', False, False)
self.allMetricsObservers.append(MetricsMessageObserver(clNetworkAPI2.chordNode))
self.assertTrue(status, 'Could not build client node')
# Wait for flooding to reach all the nodes
waiter = ConnectivityCounter()
yield waiter.waitForConnectivity(3, clNetworkAPI.chordNode) # Does not count clNode itself.
# Now shut everything down
yield clNetworkAPI.disconnect()
yield clNetworkAPI2.disconnect()
yield bsNetworkAPI.disconnect()
yield bsNetworkAPI2.disconnect()
if Config.USE_CONNECTION_CACHE:
yield TestUtils.waitForConnectionCache()
else:
yield TestUtils.wait(5)
defer.returnValue(True)
def testAggregationMessage(self):
'''
Send flooding message and ask for aggregation response.
'''
print("testAggregationMessage method starting...")
counter = 1
try:
d = defer.Deferred()
messageNum = random.randint(1,9999999)
# Reset message count
self.observer.resetMessageCount()
self.observer.printDebug = True
aggMax = numNodes + 2
#aggNumber = 3
for aggNumber in range(1, aggMax+1):
#if aggNumber == aggNumber:
# Flood from bootstrap to all nodes
for i in range(numMessages):
#d.addCallback(self.sendFlood, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber)
yield self.sendFlood(None, self.bsNode,messageNum+i,'theEnclave', data="SEND_AGG_RESPONSE:%d" % aggNumber)
print("sending message %d", counter)
counter += 1
d.callback(True)
yield d
# Wait a few seconds
yield TestUtils.wait(5)
print("testAggregationMessage check results...")
# Now check the results
yield self.checkResults(aggMax)
except Exception, e:
self.anError(e)
def test_scripts():
'''yield list of scripts to test.'''
# the current directory
current_dir = os.getcwd()
# directory location of tests
testing_dir = TestUtils.get_tests_directory()
# directory location of scripts
scripts_dir = os.path.join(os.path.dirname(testing_dir), "CGAT", "scripts")
# directories with tests (correspond to script names and
# hence end in .py)
test_dirs = glob.glob(os.path.join(testing_dir, "*.py"))
# the config file
config_file = os.path.join(testing_dir, "_test_scripts.yaml")
if os.path.exists(config_file):
config = yaml.load(open(config_file))
if config is not None:
if "restrict" in config and config["restrict"]:
values = config["restrict"]
if "glob" in values:
test_dirs = os.path.join(testing_dir, values["glob"])
if "manifest" in values:
# take scripts defined in the MANIFEST.in file
test_dirs = [
x for x in open("MANIFEST.in") if
x.startswith("include scripts") and x.endswith(".py\n")
]
test_dirs = [
re.sub("include\s*scripts/", "tests/", x[:-1])
for x in test_dirs
]
if "regex" in values:
rx = re.compile(values["regex"])
test_dirs = list(filter(rx.search, test_dirs))
# ignore those which don't exist as tests (files added through MANIFEST.in,
# such as version.py, __init__.py, ...
test_dirs = [x for x in test_dirs if os.path.exists(x)]
# ignore non-directories
test_dirs = [x for x in test_dirs if os.path.isdir(x)]
test_dirs.sort()
# restrict tests run according to chunk parameters
starting_test_number = os.getenv('CGAT_TASK_ID', None)
test_increment = os.getenv('CGAT_TASK_STEPSIZE', None)
try:
starting_test_number, test_increment = \
(int(starting_test_number) - 1,
int(test_increment))
test_dirs = test_dirs[starting_test_number:starting_test_number +
test_increment]
except TypeError:
pass
for test_script in test_dirs:
script_name = os.path.basename(test_script)
check_main.description = os.path.join(script_name, "def_main")
yield (check_main,
os.path.abspath(os.path.join(scripts_dir, script_name)))
fn = '%s/tests.yaml' % test_script
if not os.path.exists(fn):
continue
script_tests = yaml.load(open(fn))
for test, values in sorted(list(script_tests.items())):
check_script.description = os.path.join(script_name, test)
if "skip_python" in values:
versions = [
x.strip() for x in str(values["skip_python"]).split(",")
]
versions = [
x for x in versions if PYTHON_VERSION.startswith(x)
]
if len(versions) > 0:
continue
if "skip_travis" in values and TRAVIS:
continue
if "skip_jenkins" in values and JENKINS:
continue
# deal with scripts in subdirectories. These are prefixed
# by a "<subdir>_" for example: optic_compare_projects.py
# is optic/compare_procjets.py
if "_" in script_name:
parts = script_name.split("_")
if os.path.exists(
os.path.join("scripts", parts[0],
"_".join(parts[1:]))):
script_name = os.path.join(parts[0], "_".join(parts[1:]))
yield (check_script, test,
os.path.abspath(os.path.join(scripts_dir, script_name)),
values.get('stdin',
None), values['options'], values['outputs'],
values['references'], test_script, current_dir)
def tearDown(self):
dirToRemove = "{0}/tmp".format(TestUtils.CreateAndGetTMPDir())
if os.path.exists(dirToRemove):
shutil.rmtree(dirToRemove)
def setUp(self):
tmpFolder = "{0}/tmp".format(TestUtils.CreateAndGetTMPDir())
if not os.path.exists(tmpFolder):
os.makedirs(tmpFolder)
def tearDownClass():
TestUtils.RemoteTMPDir()
def parse_options(argv):
"""
Parses and checks the command-line options.
Returns:
A tuple containing the options structure.
"""
usage = 'Usage: %prog [options]'
desc = 'Example: %prog -i flatbufFile -d'
parser = optparse.OptionParser(usage=usage, description=desc)
parser.add_option('-i',
'--flatbuf',
dest='input_file',
action='store',
help='Test flatbuf',
metavar='PROTOTEXT_FILE')
parser.add_option('-o',
'--output',
dest='output_dir',
action='store',
default='./',
help='Output directory',
metavar='OUTPUT_DIR')
parser.add_option('-d',
'--debug',
action="store_true",
dest="log_level",
default=False,
help='Log Level')
parser.add_option('-p',
'--port',
action="store",
dest="port_addr",
default=6666,
help='Client Port.')
parser.add_option('-s',
'--doNotShutServer',
action="store_false",
dest="shut_server",
default=False,
help='Don\'t Shut Down Server.')
parser.add_option('--image',
'--img',
dest='image_file',
action='store',
default=None,
help='Image file')
options, categories = parser.parse_args(argv[1:])
_input = options.input_file
outfile_list = list()
if options.input_file is None:
parser.error('Input is missing')
else:
options.input_file = os.path.abspath(options.input_file)
if not os.path.exists(options.input_file):
print("Invalid input directory: ", options.input_file)
sys.exit(1)
else:
options.input_file = tu.get_input_files(options.input_file)
if options.image_file is None:
print("No img to run")
else:
options.image_file = os.path.abspath(options.image_file)
if not os.path.exists(options.image_file):
print("Invalid image location: ", options.image_file)
sys.exit(1)
else:
options.image_file = tu.get_input_files(options.image_file)
return (options, categories)
def main():
seqs = cfg.sequences[cfg.hevc_A] + cfg.sequences[cfg.hevc_B]
in_names = cfg.class_sequence_names[cfg.hevc_A] + cfg.class_sequence_names[
cfg.hevc_B]
ver = 20
shared_param = ("--preset", "ultrafast", "--gop", "0", "--no-tmvp")
thread_range = (13, 14, 15, 16) #(4,6,8,10,12,14)
owf_range = (1, 2, 3) #(2,4,8,16)
outname = "skvz_thread_owf_test_v{}".format(ver)
# Set shared param
tpg_scal = TU.TestParameterGroup()
tpg_scal.add_const_param(version = ver,
bin_name = cfg.skvz_ver_bin.format(ver),
input_names = in_names,
layer_args = shared_param)\
.add_param_set(_thrd = thread_range,
_owf = owf_range)
tpg_sim = tpg_scal.copy()
# Set scalable param
round2 = lambda x, base=2: int(base * round(x / base))
strscale = lambda size, scale: "x".join(
map(lambda x: str(round2(int(x) * scale)),
re.search("_*(\d+)x(\d+)[_.]*", size).group(1, 2)))
seq_scale = lambda scale, st: (st[0], strscale(st[1], scale), st[2])
seq_map = lambda scale, seq: r"{}_{}_{}_zerophase_0.9pi.yuv".format(
*seq_scale(
scale,
re.search("(.+\\\\.+)_(\d+x\d+)_(\d+)[_.]", seq).group(1, 2, 3)))
bl_seq_map = lambda scale: lambda seq: (seq_map(scale[0], seq[0]), )
scal_seq_map = lambda scale: lambda seq: (seq_map(scale[0], seq[
0]), ) + seq #TODO: Add scaling to el seq?
scal_seqs = tuple(map(scal_seq_map((0.5, 1)), seqs))
tpg_scal.add_const_param(inputs=scal_seqs)
tpg_scal.set_param_group_transformer(
TU.transformerFactory(
test_name=lambda *, _thrd, _owf, **param: "SCAL_THRD{}_OWF{}".
format(_thrd, _owf),
layer_args=lambda *, layer_args, _thrd, _owf, **param:
(layer_args + ("--threads", str(_thrd)) +
("--owf", str(_owf)), layer_args + ("--threads", str(_thrd)) +
("--owf", str(_owf)))))
# Set simulcast param
tpg_sim.add_const_param(inputs = seqs)\
.add_param_set(_layer=("BL","EL"))
tpg_sim.set_param_group_transformer(
TU.transformerFactory(
test_name=lambda *, _layer, _thrd, _owf, **param: "{}_THRD{}_OWF{}"
.format(_layer, _thrd, _owf),
layer_args=lambda *, layer_args, _thrd, _owf, **param:
(layer_args + ("--threads", str(_thrd)) + ("--owf", str(_owf)), ),
inputs=lambda *, inputs, _layer, **param: tuple(
map(bl_seq_map((0.5, )), inputs))
if _layer in "BL" else inputs))
#Run tests
tests_scal = tpg_scal.to_skvz_test_instance()
tests_sim = tpg_sim.to_skvz_test_instance()
combi = TU.generate_combi(tpg_sim,
combi_cond=TU.combiFactory(
_thrd=op.eq,
_owf=op.eq,
_layer=lambda p1, p2: 0
if p1 == p2 else (-1 if p1 == "BL" else 1)))
sim_names = TU.get_combi_names(combi)
test_names = TU.get_test_names(tests_scal) + sim_names
matrix_summary = TU.make_BDBRMatrix_definition(
test_names + TU.get_test_names(tests_sim),
write_bdbr=True,
write_bits=False,
write_psnr=False,
layering_func=lambda t: (-1, 1) if "SCAL" not in t else (-1, ),
filter_func=lambda t: True
if ("BL" in t and "EL" in t) or "SCAL" in t else False)
anchor_summary = TU.make_AnchorList_multiAnchor_definition(
test_names,
global_filter=lambda t: True if "SCAL" in t else False,
bdbr_anchor_func=TU.anchorFuncFactory_match_layer(lambda t: tuple(
a for a in sim_names
if (t.split(sep='_')[1] in a) and (t.split(sep='_')[2] in a))),
bdbr_layer_func=TU.layerFuncFactory([
[None, 1],
]),
time_anchor_func=TU.anchorFuncFactory_match_layer(lambda t: (
None, ) + tuple((a, l) if l >= 0 else a for a in sim_names
if (t.split(sep='_')[1] in a) and
(t.split(sep='_')[2] in a) for l in [-1, 1])))
summaries = {sn_BDBRM: matrix_summary, sn_ANCHOR: anchor_summary}
runTests(tests_scal + tests_sim, outname, layer_combi=combi, **summaries)
def teardown_module(module):
server.terminate()
tu.stopNet("scimma-test")
def test_python_len_and_dictionary_size_equality(self, lst):
dictionary = Dictionary()
lst = TestUtils.lst_validate(lst)
dictionary.from_list(lst)
self.assertEqual(len(dictionary.to_list()), dictionary.size()[1])
def tearDown(self):
TestUtils.RemoteTMPDir()
def setup_module(module):
global server
tu.startNet("scimma-test")
server = tu.ServerContainer(tu.testTag(),
opts=["-v", "shared:/root/shared"])
server.start()
def main():
options, categories = parse_options(sys.argv)
clientlogfile = options.output_dir + "/DLAClientLogger.log"
resultsdir = options.output_dir + "/results/"
tu.logger_setup(options.log_level, clientlogfile)
dlasocket = ds.dlaSocket()
#Set Client Port
dlasocket.setPort(options.port_addr)
dlasocket.connect(dlasocket.HOST, dlasocket.PORT)
dlasocket.setTimeout(dlasocket.getTimeout())
logging.info("DLA Client open at PORT: {0}.".format(dlasocket.getPort()))
getWelcomeMessage(dlasocket)
test_i = 0
while test_i < len(options.input_file):
fbuf_file = options.input_file[test_i]
fbuf_size = os.stat(fbuf_file).st_size
if options.image_file is None:
fbuf_file_name = options.input_file[test_i].split("/")[-1]
logging.info("Attempting to read flatbuf: [{0}], " \
"size[{1}].".format(fbuf_file_name, fbuf_size))
readFlatbuf(dlasocket, fbuf_file)
logging.info("Attempting to run flatbuf: [{0}], " \
"size[{1}].".format(fbuf_file_name, fbuf_size))
runFlatbuf(dlasocket, fbuf_file)
else:
img_file = options.image_file[test_i]
img_size = os.stat(img_file).st_size
fbuf_file_name = options.input_file[test_i].split("/")[-1]
logging.info("Attempting to read flatbuf: [{0}], " \
"size[{1}].".format(fbuf_file_name, fbuf_size))
readFlatbuf(dlasocket, fbuf_file)
image_file_name = options.image_file[test_i].split("/")[-1]
logging.info("Attempting to run image: [{0}], " \
"size[{1}].".format(image_file_name, img_size))
runImage(dlasocket, img_file)
numOutputs = getNumOutputs(dlasocket)
for ii in range(numOutputs):
writeOutput(dlasocket, ii, resultsdir)
test_i += 1
#Send ShutDown command to Server if shut_server is True.
if options.shut_server:
shutDownServer(dlasocket)
#Close dlsSocket
dlasocket.closeConnection()
return 0
def _getOutDir(self):
return TestUtils.CreateAndGetTMPDir()
def main(agrv = None):
#FLAGS.para_name
if FLAGS.Mode == 'Train':
#keep_probability = tf.placeholder(tf.float32, name = "keep_prob") #dropout: keep_probability
is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining
image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img')
annotation = tf.placeholder(tf.int32, shape = [None, SP_HEIGHT, SP_WIDTH, 1], name = 'annotation')
Testloss = tf.placeholder(tf.float32, name = 'Test_loss')
logits, pred = net.PSPUnet(image, is_training)
tf.summary.image("image", image, max_outputs = batch_sz)
tf.summary.image("groud_truth", tf.cast(annotation, tf.uint8), max_outputs = batch_sz)
tf.summary.image("pred_annotation", tf.cast(pred, tf.uint8), max_outputs = batch_sz)
#softmax cross entropy loss
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits,
labels = tf.squeeze(annotation, axis=[3]),
name = 'entropy_loss'))
DSC_batch = tf.placeholder(tf.float32, name = 'Dice_coeff')
trainable_var = tf.trainable_variables()
if FLAGS.Debug:
for var in trainable_var:
tf.summary.histogram(var.op.name, var)
tf.add_to_collection('reg_loss', tf.nn.l2_loss(var))
#BN: update moving_mean&moving_variance
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = Train(loss, trainable_var)
# loss_train_op = tf.summary.scalar('training_loss', loss)
tf.summary.scalar('training_loss', loss)
summary_op = tf.summary.merge_all()
loss_test_op = tf.summary.scalar('test_loss',Testloss)
DSC_op = tf.summary.scalar('Dice_coefficient', DSC_batch)
print("setting up image reader...")
image_batch_a, label_batch_a = read_mat.read_and_decord('Train')
img_train_batch, label_train_batch = tf.train.shuffle_batch([image_batch_a, label_batch_a],
batch_size=batch_sz, capacity=batch_sz*2, min_after_dequeue=batch_sz)
print (img_train_batch.shape)
print ('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
#filename = 'E:/maning/Cells/mod/model-87500'
#saver.restore(sess,filename)
summary_writer = tf.summary.FileWriter(FLAGS.logs_dir,sess.graph)
sess.run(tf.global_variables_initializer())
threads = tf.train.start_queue_runners(sess)
test_i = -1
for itr in range (MAX_ITERATION):
#img_batch shape:[batch_size, depth, height, width, chanl], type: ndarray
img_batch, l_batch = sess.run([img_train_batch, label_train_batch ])
print (itr, img_batch.shape)
feed = {image: img_batch, annotation: l_batch, is_training: True}
sess.run(train_op, feed_dict = feed)
train_loss_print, summary_str= sess.run([loss,summary_op], feed_dict = feed)
print (train_loss_print)
summary_writer.add_summary(summary_str, itr)
if itr%test_epoch == 0 or itr%5000 == 0:
saver.save(sess, './mod/model', global_step=itr)
elif itr == (MAX_ITERATION - 1):
saver.save(sess, './mod/model', global_step=itr)
############################# test test test test test test test test test test#####################################
if (itr!=0 and itr%test_epoch == 0) or itr == (MAX_ITERATION - 1):
#overfittest
test_i = test_i+1
print("train finish! start test~")
test_img_data_a, test_label_data_a = read_mat.read_and_decord('Test')
test_img_train_batch, test_label_train_batch = tf.train.batch([test_img_data_a, test_label_data_a],batch_size=1, capacity=1)
threads = tf.train.start_queue_runners(sess)
Testl = 0.0
for test_itr in range (TEST_RAW):
test_img_batch, test_l_batch = sess.run([test_img_train_batch, test_label_train_batch])
test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS)
test_lbl_sp = data_pro.image_splite(test_l_batch,COLS,ROWS)
pred_sp = {}
for col in range(COLS):
pred_sp[col]={}
for row in range(ROWS):
img_test = test_img_sp[col][row]
lbl_test = test_lbl_sp[col][row]
test_feed = {image: img_test, annotation: lbl_test, is_training: False}
test_pred_logits, pred_image, testloss = sess.run([logits, pred, loss], feed_dict = test_feed)
pred_sp[col][row] = pred_image
Testl = Testl+testloss
#pred_image = sess.run(tf.argmax(input=test_pred_logits,axis=3))
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
label_batch = np.squeeze(test_l_batch)
pred_batch = np.squeeze(pred_all)
#label_batch_tp = np.transpose(label_batch, (0, 2, 1))
label_tosave = np.reshape(label_batch, [HEIGHT,WIDTH])
#pred_batch_tp = np.transpose(pred_batch, (0, 2, 1))
pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH])
print("test_itr:",test_itr)
# tep = test_pred_annotation[0, 30, :, 0]
#np.savetxt('pred30.csv', tep, delimiter=',')
#np.savetxt('dice_smi_co.csv',test_dice_coe, delimiter=',')
utils.save_imgs(test_itr, label_tosave, pred_tosave, itr)
Testl = Testl/TEST_RAW
test_summary_str = sess.run(loss_test_op, feed_dict = {Testloss:Testl})
print (test_i,':',Testl)
summary_writer.add_summary(test_summary_str, test_i)
# Dise similarity coefficient
DSC = utils.Accuracy_Measure(itr)
DSC_Summary_str = sess.run(DSC_op, feed_dict = {DSC_batch:DSC})
print (test_i,':',DSC)
summary_writer.add_summary(DSC_Summary_str, test_i)
elif FLAGS.Mode == 'Visualize':
start = time.time()
is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining
image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img')
logits, pred = net.PSPUnet(image, is_training)
print ('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
sess.run(tf.global_variables_initializer())
saver.restore(sess,modelname)
img_folder = img_path
test_itr=0
for imgname in os.listdir(img_folder):
test_itr = test_itr+1
print('path is :', img_folder+imgname)
test_img_batch = testutils.read_img(img_folder+imgname)
test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS)
pred_sp = {}
logits_sp = {}
for col in range(COLS):
pred_sp[col]={}
logits_sp[col] = {}
for row in range(ROWS):
img_test = test_img_sp[col][row]
test_feed = {image: img_test, is_training: False}
test_pred_logits, pred_image = sess.run([logits, pred], feed_dict = test_feed)
logits_sp[col][row] = test_pred_logits
pred_sp[col][row] = pred_image
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
pred_batch = np.squeeze(pred_all)
print("test_itr:",test_itr)
testutils.saveImage(img_folder+imgname, pred_batch, imgname, test_save_path)
end = time.time()
elapse = end - start
print('elapse time is :', elapse)
print ('finished!')
