def dense_solve_prep(model, dataBasics, string, prevSpan, prevNumSpan, span,
numSpan, cp, input, wv, bv, activations):
# filters can be seen as the output of a convolutional layer
nfilters = basics.numberOfFilters(wv)
# features can be seen as the inputs for a convolutional layer
nfeatures = basics.numberOfFeatures(wv)
# space holders for computation values
biasCollection = {}
filterCollection = {}
for ((p1, c1), (p, c), w) in wv:
if c1 - 1 in range(nfeatures) and c - 1 in range(nfilters):
filterCollection[c1 - 1, c - 1] = w
for (p, c, w) in bv:
if c - 1 in range(nfilters):
for l in range(nfeatures):
biasCollection[l, c - 1] = w
(bl1, newInput) = dense_safety_solve(nfeatures, nfilters, filterCollection,
biasCollection, input, activations,
prevSpan, prevNumSpan, span, numSpan,
cp)
basics.nprint("completed a round of processing ")
return (bl1, newInput)
def bestChild(self, index):
allValues = {}
for childIndex in self.children[index]:
allValues[childIndex] = self.cost[childIndex] / float(
self.numberOfVisited[childIndex])
basics.nprint("finding best children from %s" % (allValues))
return max(allValues.iteritems(), key=operator.itemgetter(1))[0]
def sampling(self, index, availableActions):
basics.nprint("start sampling node %s" % (index))
availableActions2 = copy.deepcopy(availableActions)
#print(availableActions,self.keypoint[index],self.indexToActionID[index])
availableActions2[self.keypoint[index]].pop(
self.indexToActionID[index], None)
sampleValues = []
i = 0
for i in range(cfg.MCTS_multi_samples):
self.spansPath = self.spans[index]
self.numSpansPath = self.numSpans[index]
self.depth = 0
self.availableActionIDs = {}
for k in self.keypoints.keys():
self.availableActionIDs[k] = availableActions2[k].keys()
self.usedActionIDs = {}
for k in self.keypoints.keys():
self.usedActionIDs[k] = []
self.accDims = []
self.d = 2
(childTerminated, val) = self.sampleNext(self.keypoint[index])
sampleValues.append(val)
#if childTerminated == True: break
i += 1
return (childTerminated, max(sampleValues))
def initialiseExplorationNode(self, index, availableActions):
basics.nprint("expanding %s" % (index))
for (actionId, (span, numSpan, _)) in availableActions.iteritems():
self.indexToNow += 1
self.indexToActionID[self.indexToNow] = actionId
self.initialiseLeafNode(self.indexToNow, index, span, numSpan)
self.children[index].append(self.indexToNow)
self.fullyExpanded[index] = True
self.usedActionsID = []
return self.children[index]
def backPropagation(self, index, value):
self.cost[index] += value
self.numberOfVisited[index] += 1
if self.parent[index] in self.parent:
basics.nprint(
"start backPropagating the value %s from node %s, whose parent node is %s"
% (value, index, self.parent[index]))
self.backPropagation(self.parent[index], value)
else:
basics.nprint("backPropagating ends on node %s" % (index))
def initialiseLeafNode(self, index, parentIndex, newSpans, newNumSpans):
basics.nprint("initialising a leaf node %s from the node %s" %
(index, parentIndex))
self.spans[index] = basics.mergeTwoDicts(self.spans[parentIndex],
newSpans)
self.numSpans[index] = basics.mergeTwoDicts(self.numSpans[parentIndex],
newNumSpans)
self.cost[index] = 0
self.parent[index] = parentIndex
self.children[index] = []
self.fullyExpanded[index] = False
self.numberOfVisited[index] = 0
def terminatedByControlledSearch(self, index):
activations1 = applyManipulation(self.activations, self.spans[index],
self.numSpans[index])
(distMethod, distVal) = cfg.controlledSearch
if distMethod == "euclidean":
dist = basics.euclideanDistance(activations1, self.activations)
elif distMethod == "L1":
dist = basics.l1Distance(activations1, self.activations)
elif distMethod == "Percentage":
dist = basics.diffPercent(activations1, self.activations)
elif distMethod == "NumDiffs":
dist = basics.diffPercent(activations1, self.activations)
basics.nprint("terminated by controlled search")
return dist > distVal
def conv_solve_prep(model, dataBasics, string, originalLayer2Consider,
layer2Consider, prevSpan, prevNumSpan, span, numSpan, cp,
input, wv, bv, activations):
# filters can be seen as the output of a convolutional layer
nfilters = basics.numberOfFilters(wv)
# features can be seen as the inputs for a convolutional layer
nfeatures = basics.numberOfFeatures(wv)
# space holders for computation values
biasCollection = {}
filterCollection = {}
for l in range(nfeatures):
for k in range(nfilters):
filter = [
w for ((p1, c1), (p, c), w) in wv if c1 == l + 1 and c == k + 1
]
bias = [w for (p, c, w) in bv if c == k + 1]
if len(filter) == 0 or len(bias) == 0:
print "error: bias =" + str(bias) + "\n filter = " + str(
filter)
else:
filter = filter[0]
bias = bias[0]
# flip the filter for convolve
flipedFilter = np.fliplr(np.flipud(filter))
biasCollection[l, k] = bias
filterCollection[l, k] = flipedFilter
#print filter.shape
input2 = copy.deepcopy(input)
if originalLayer2Consider > layer2Consider:
(bl1, newInput) = conv_safety_solve(layer2Consider, nfeatures,
nfilters, filterCollection,
biasCollection, input, activations,
prevSpan, prevNumSpan, span,
numSpan, cp)
else:
(bl1, newInput) = conv_safety_solve(layer2Consider, nfeatures,
nfilters, filterCollection,
biasCollection, input, activations,
prevSpan, prevNumSpan, span,
numSpan, cp)
basics.nprint("completed a round of processing of the entire image ")
return (bl1, newInput)
def initialiseLeafNode(self, index, parentIndex, newSpans, newNumSpans):
basics.nprint("initialising a leaf node %s from the node %s" %
(index, parentIndex))
self.spans[index] = basics.mergeTwoDicts(self.spans[parentIndex],
newSpans)
self.numSpans[index] = basics.mergeTwoDicts(self.numSpans[parentIndex],
newNumSpans)
self.cost[index] = 0
self.parent[index] = parentIndex
self.children[index] = []
self.fullyExpanded[index] = False
self.numberOfVisited[index] = 0
activations1 = applyManipulation(self.activations, self.spans[index],
self.numSpans[index])
self.re_training.addDatum(activations1, self.originalClass,
self.originalClass)
def treeTraversal(self, index):
if self.fullyExpanded[index] == True:
basics.nprint("tree traversal on node %s" % (index))
allValues = {}
for childIndex in self.children[index]:
allValues[childIndex] = (
self.cost[childIndex] /
float(self.numberOfVisited[childIndex])
) + explorationRate * math.sqrt(
math.log(self.numberOfVisited[index]) /
float(self.numberOfVisited[childIndex]))
#nextIndex = max(allValues.iteritems(), key=operator.itemgetter(1))[0]
if self.player_mode == "adversary" and self.keypoint[index] == 0:
allValues2 = {}
for k, v in allValues.iteritems():
allValues2[k] = 1 / float(allValues[k])
nextIndex = np.random.choice(allValues.keys(),
1,
p=[
x / sum(allValues2.values())
for x in allValues2.values()
])[0]
else:
nextIndex = np.random.choice(allValues.keys(),
1,
p=[
x / sum(allValues.values())
for x in allValues.values()
])[0]
if self.keypoint[index] in self.usedActionsID.keys(
) and self.keypoint[index] != 0:
self.usedActionsID[self.keypoint[index]].append(
self.indexToActionID[index])
elif self.keypoint[index] != 0:
self.usedActionsID[self.keypoint[index]] = [
self.indexToActionID[index]
]
return self.treeTraversal(nextIndex)
else:
basics.nprint("tree traversal terminated on node %s" % (index))
availableActions = copy.deepcopy(self.actions)
for k in self.usedActionsID.keys():
for i in self.usedActionsID[k]:
availableActions[k].pop(i, None)
return (index, availableActions)
def treeTraversal(self, index):
if self.fullyExpanded[index] == True:
basics.nprint("tree traversal on node %s" % (index))
allValues = {}
for childIndex in self.children[index]:
allValues[childIndex] = (self.cost[childIndex] / float(
self.numberOfVisited[childIndex])) + cp * math.sqrt(
math.log(self.numberOfVisited[index]) /
float(self.numberOfVisited[childIndex]))
nextIndex = max(allValues.iteritems(),
key=operator.itemgetter(1))[0]
self.usedActionsID.append(self.indexToActionID[nextIndex])
return self.treeTraversal(nextIndex)
else:
basics.nprint("tree traversal terminated on node %s" % (index))
availableActions = copy.deepcopy(self.actions)
for i in self.usedActionsID:
availableActions.pop(i, None)
return (index, availableActions)
def diffImage(image1, image2):
i = 0
if len(image1.shape) == 1:
for x in range(len(image1)):
if image1[x] != image2[x]:
i += 1
basics.nprint("dimension %s is changed from %s to %s" %
(x, image1[x], image2[x]))
elif len(image1.shape) == 2:
for x in range(len(image1)):
for y in range(len(image1[0])):
if image1[x][y] != image2[x][y]:
i += 1
basics.nprint(
"dimension (%s,%s) is changed from %s to %s" %
(x, y, image1[x][y], image2[x][y]))
elif len(image1.shape) == 3:
for x in range(len(image1)):
for y in range(len(image1[0])):
for z in range(len(image1[0])):
if image1[x][y][z] != image2[x][y][z]:
i += 1
basics.nprint(
"dimension (%s,%s,%s) is changed from %s to %s" %
(x, y, z, image1[x][y][z], image2[x][y][z]))
print("%s elements have been changed!" % i)
def initialiseExplorationNode(self, index, availableActions):
basics.nprint("expanding %s" % (index))
if self.keypoint[index] != 0:
for (
actionId, (span, numSpan, _)
) in availableActions[self.keypoint[index]].iteritems(
): #initialisePixelSets(self.model,self.image,list(set(self.spans[index].keys() + self.usefulPixels))):
self.indexToNow += 1
self.keypoint[self.indexToNow] = 0
self.indexToActionID[self.indexToNow] = actionId
self.initialiseLeafNode(self.indexToNow, index, span, numSpan)
self.children[index].append(self.indexToNow)
else:
for kp in self.keypoints.keys(
): #initialisePixelSets(self.model,self.image,list(set(self.spans[index].keys() + self.usefulPixels))):
self.indexToNow += 1
self.keypoint[self.indexToNow] = kp
self.indexToActionID[self.indexToNow] = 0
self.initialiseLeafNode(self.indexToNow, index, {}, {})
self.children[index].append(self.indexToNow)
self.fullyExpanded[index] = True
self.usedActionsID = {}
return self.children[index]
def initialiseActions(self):
# initialise actions according to the type of manipulations
if self.manipulationType == "sift_twoPlayer":
actions = initialiseSiftKeypointsTwoPlayer(self.autoencoder,
self.activations, [])
self.keypoints[0] = 0
i = 1
for k in actions[0]:
self.keypoints[i] = k
i += 1
#print self.keypoints
else:
print("unknown manipulation type")
exit
#print("actions=%s"%(actions.keys()))
for i in range(len(actions)):
ast = {}
for j in range(len(actions[i])):
ast[j] = actions[i][j]
self.actions[i] = ast
basics.nprint("%s actions have been initialised. " %
(len(self.actions)))
def handleOne(model, dataset, dc, imgIdx):
print(imgIdx)
# get an image to interpolate
image = dataset.getTestImage(imgIdx)
print("the shape of the input is " + str(image.shape))
if cfg.dataset == "twoDcurve": image = np.array([3.58747339, 1.11101673])
dc.initialiseIndex(imgIdx)
originalImage = copy.deepcopy(image)
if cfg.checkingMode == "stepwise":
k = cfg.startLayer
elif cfg.checkingMode == "specificLayer":
k = cfg.maxLayer
while k <= cfg.maxLayer:
layerType = model.getLayerType(k)
re = False
start_time = time.time()
# only these layers need to be checked
if layerType in ["Convolution2D", "Conv2D", "Dense", "InputLayer"
] and k >= 0:
dc.initialiseLayer(k)
st = SearchTree(image, k)
st.addImages(model, [image])
print(
"\n================================================================"
)
print "\nstart checking the safety of layer " + str(k)
(originalClass, originalConfident) = model.predict(image)
origClassStr = dataset.labels(int(originalClass))
path0 = "%s/%s_original_as_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imgIdx, origClassStr,
originalConfident)
dataset.save(-1, originalImage, path0)
# for every layer
f = 0
while f < cfg.numOfFeatures:
f += 1
print(
"\n================================================================"
)
print("Round %s of layer %s for image %s" % (f, k, imgIdx))
index = st.getOneUnexplored()
imageIndex = copy.deepcopy(index)
# for every image
# start from the first hidden layer
t = 0
re = False
while True and index != (-1, -1):
# pick the first element of the queue
print "(1) get a manipulated input ..."
(image0, span, numSpan, numDimsToMani,
_) = st.getInfo(index)
print "current layer: %s." % (t)
print "current index: %s." % (str(index))
path2 = cfg.directory_pic_string + "/temp.png"
print "current operated image is saved into %s" % (path2)
dataset.save(index[0], image0, path2)
print "(2) synthesise region from %s..." % (span.keys())
# ne: next region, i.e., e_{k+1}
#print "manipulated: %s"%(st.manipulated[t])
(nextSpan, nextNumSpan,
numDimsToMani) = regionSynth(model, cfg.dataset, image0,
st.manipulated[t], t, span,
numSpan, numDimsToMani)
st.addManipulated(t, nextSpan.keys())
(nextSpan, nextNumSpan,
npre) = precisionSynth(model, image0, t, span, numSpan,
nextSpan, nextNumSpan)
print "dimensions to be considered: %s" % (nextSpan)
print "spans for the dimensions: %s" % (nextNumSpan)
if t == k:
# only after reaching the k layer, it is counted as a pass
print "(3) safety analysis ..."
# wk for the set of counterexamples
# rk for the set of images that need to be considered in the next precision
# rs remembers how many input images have been processed in the last round
# nextSpan and nextNumSpan are revised by considering the precision npre
(nextSpan, nextNumSpan, rs, wk,
rk) = safety_analysis(model, dataset, t, imgIdx, st,
index, nextSpan, nextNumSpan,
npre)
if len(rk) > 0:
rk = (zip(*rk))[0]
print "(4) add new images ..."
random.seed(time.time())
if len(rk) > numOfPointsAfterEachFeature:
rk = random.sample(
rk, numOfPointsAfterEachFeature)
diffs = basics.diffImage(image0, rk[0])
print(
"the dimensions of the images that are changed in the this round: %s"
% diffs)
if len(diffs) == 0:
st.clearManipulated(k)
return
st.addImages(model, rk)
st.removeProcessed(imageIndex)
(re, percent, eudist, l1dist,
l0dist) = reportInfo(image, wk)
print "euclidean distance %s" % (
basics.euclideanDistance(image, rk[0]))
print "L1 distance %s" % (basics.l1Distance(
image, rk[0]))
print "L0 distance %s" % (basics.l0Distance(
image, rk[0]))
print "manipulated percentage distance %s\n" % (
basics.diffPercent(image, rk[0]))
break
else:
st.removeProcessed(imageIndex)
break
else:
print "(3) add new intermediate node ..."
index = st.addIntermediateNode(image0, nextSpan,
nextNumSpan, npre,
numDimsToMani, index)
re = False
t += 1
if re == True:
dc.addManipulationPercentage(percent)
print "euclidean distance %s" % (eudist)
print "L1 distance %s" % (l1dist)
print "L0 distance %s" % (l0dist)
print "manipulated percentage distance %s\n" % (percent)
dc.addEuclideanDistance(eudist)
dc.addl1Distance(l1dist)
dc.addl0Distance(l0dist)
(ocl, ocf) = model.predict(wk[0])
dc.addConfidence(ocf)
break
if f == cfg.numOfFeatures:
print "(6) no adversarial example is found in this layer within the distance restriction."
st.destructor()
elif layerType in ["Input"
] and k < 0 and mcts_mode == "sift_twoPlayer":
print "directly handling the image ... "
dc.initialiseLayer(k)
(originalClass, originalConfident) = model.predict(image)
origClassStr = dataset.labels(int(originalClass))
path0 = "%s/%s_original_as_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imgIdx, origClassStr,
originalConfident)
dataset.save(-1, originalImage, path0)
# initialise a search tree
st = MCTSTwoPlayer(model, model, image, image, -1, "cooperator",
dataset)
st.initialiseActions()
st.setManipulationType("sift_twoPlayer")
start_time_all = time.time()
runningTime_all = 0
numberOfMoves = 0
while st.terminalNode(
st.rootIndex
) == False and st.terminatedByControlledSearch(
st.rootIndex
) == False and runningTime_all <= cfg.MCTS_all_maximal_time:
print("the number of moves we have made up to now: %s" %
(numberOfMoves))
eudist = st.euclideanDist(st.rootIndex)
l1dist = st.l1Dist(st.rootIndex)
l0dist = st.l0Dist(st.rootIndex)
percent = st.diffPercent(st.rootIndex)
diffs = st.diffImage(st.rootIndex)
print("euclidean distance %s" % (eudist))
print("L1 distance %s" % (l1dist))
print("L0 distance %s" % (l0dist))
print("manipulated percentage distance %s" % (percent))
print("manipulated dimensions %s" % (diffs))
start_time_level = time.time()
runningTime_level = 0
childTerminated = False
while runningTime_level <= cfg.MCTS_level_maximal_time:
(leafNode,
availableActions) = st.treeTraversal(st.rootIndex)
newNodes = st.initialiseExplorationNode(
leafNode, availableActions)
for node in newNodes:
(childTerminated,
value) = st.sampling(node, availableActions)
#if childTerminated == True: break
st.backPropagation(node, value)
#if childTerminated == True: break
runningTime_level = time.time() - start_time_level
basics.nprint("best possible one is %s" %
(str(st.bestCase)))
bestChild = st.bestChild(st.rootIndex)
#st.collectUselessPixels(st.rootIndex)
st.makeOneMove(bestChild)
image1 = st.applyManipulationToGetImage(
st.spans[st.rootIndex], st.numSpans[st.rootIndex])
diffs = st.diffImage(st.rootIndex)
path0 = "%s/%s_temp_%s.png" % (cfg.directory_pic_string,
imgIdx, len(diffs))
dataset.save(-1, image1, path0)
(newClass, newConfident) = model.predict(image1)
print("confidence: %s" % (newConfident))
if childTerminated == True: break
# store the current best
(_, bestSpans, bestNumSpans) = st.bestCase
image1 = st.applyManipulationToGetImage(
bestSpans, bestNumSpans)
path0 = "%s/%s_currentBest.png" % (cfg.directory_pic_string,
imgIdx)
dataset.save(-1, image1, path0)
numberOfMoves += 1
runningTime_all = time.time() - start_time_all
(_, bestSpans, bestNumSpans) = st.bestCase
#image1 = applyManipulation(st.image,st.spans[st.rootIndex],st.numSpans[st.rootIndex])
image1 = st.applyManipulationToGetImage(bestSpans, bestNumSpans)
(newClass, newConfident) = model.predict(image1)
newClassStr = dataset.labels(int(newClass))
re = newClass != originalClass
if re == True:
path0 = "%s/%s_%s_%s_modified_into_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imgIdx, "sift_twoPlayer",
origClassStr, newClassStr, newConfident)
dataset.save(-1, image1, path0)
path0 = "%s/%s_diff.png" % (cfg.directory_pic_string, imgIdx)
dataset.save(-1, np.subtract(image, image1), path0)
print(
"\nfound an adversary image within prespecified bounded computational resource. The following is its information: "
)
print("difference between images: %s" %
(basics.diffImage(image, image1)))
print("number of adversarial examples found: %s" % (st.numAdv))
eudist = basics.euclideanDistance(st.image, image1)
l1dist = basics.l1Distance(st.image, image1)
l0dist = basics.l0Distance(st.image, image1)
percent = basics.diffPercent(st.image, image1)
print("euclidean distance %s" % (eudist))
print("L1 distance %s" % (l1dist))
print("L0 distance %s" % (l0dist))
print("manipulated percentage distance %s" % (percent))
print("class is changed into %s with confidence %s\n" %
(newClassStr, newConfident))
dc.addRunningTime(time.time() - start_time_all)
dc.addConfidence(newConfident)
dc.addManipulationPercentage(percent)
dc.addEuclideanDistance(eudist)
dc.addl1Distance(l1dist)
dc.addl0Distance(l0dist)
#path0="%s/%s_original_as_%s_heatmap.png"%(directory_pic_string,imgIdx,origClassStr)
#plt.imshow(GMM(image),interpolation='none')
#plt.savefig(path0)
#path1="%s/%s_%s_%s_modified_into_%s_heatmap.png"%(directory_pic_string,imgIdx,"sift_twoPlayer", origClassStr,newClassStr)
#plt.imshow(GMM(image1),interpolation='none')
#plt.savefig(path1)
else:
print(
"\nfailed to find an adversary image within prespecified bounded computational resource. "
)
elif layerType in ["Input"] and k < 0 and mcts_mode == "singlePlayer":
print "directly handling the image ... "
dc.initialiseLayer(k)
(originalClass, originalConfident) = model.predict(image)
origClassStr = dataset.labels(int(originalClass))
path0 = "%s/%s_original_as_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imgIdx, origClassStr,
originalConfident)
dataset.save(-1, originalImage, path0)
# initialise a search tree
st = SearchMCTS(model, image, k)
st.initialiseActions()
start_time_all = time.time()
runningTime_all = 0
numberOfMoves = 0
while st.terminalNode(
st.rootIndex
) == False and st.terminatedByControlledSearch(
st.rootIndex
) == False and runningTime_all <= cfg.MCTS_all_maximal_time:
print("the number of moves we have made up to now: %s" %
(numberOfMoves))
eudist = st.euclideanDist(st.rootIndex)
l1dist = st.l1Dist(st.rootIndex)
l0dist = st.l0Dist(st.rootIndex)
percent = st.diffPercent(st.rootIndex)
diffs = st.diffImage(st.rootIndex)
print "euclidean distance %s" % (eudist)
print "L1 distance %s" % (l1dist)
print "L0 distance %s" % (l0dist)
print "manipulated percentage distance %s" % (percent)
print "manipulated dimensions %s" % (diffs)
start_time_level = time.time()
runningTime_level = 0
childTerminated = False
while runningTime_level <= cfg.MCTS_level_maximal_time:
(leafNode,
availableActions) = st.treeTraversal(st.rootIndex)
newNodes = st.initialiseExplorationNode(
leafNode, availableActions)
for node in newNodes:
(childTerminated,
value) = st.sampling(node, availableActions)
if childTerminated == True: break
st.backPropagation(node, value)
if childTerminated == True: break
runningTime_level = time.time() - start_time_level
print("best possible one is %s" % (st.showBestCase()))
bestChild = st.bestChild(st.rootIndex)
#st.collectUselessPixels(st.rootIndex)
st.makeOneMove(bestChild)
image1 = applyManipulation(st.image, st.spans[st.rootIndex],
st.numSpans[st.rootIndex])
diffs = st.diffImage(st.rootIndex)
path0 = "%s/%s_temp_%s.png" % (cfg.directory_pic_string,
imgIdx, len(diffs))
dataset.save(-1, image1, path0)
(newClass, newConfident) = model.predict(image1)
print "confidence: %s" % (newConfident)
if childTerminated == True: break
# store the current best
(_, bestSpans, bestNumSpans) = st.bestCase
image1 = applyManipulation(st.image, bestSpans, bestNumSpans)
path0 = "%s/%s_currentBest.png" % (cfg.directory_pic_string,
imgIdx)
dataset.save(-1, image1, path0)
runningTime_all = time.time() - runningTime_all
numberOfMoves += 1
(_, bestSpans, bestNumSpans) = st.bestCase
#image1 = applyManipulation(st.image,st.spans[st.rootIndex],st.numSpans[st.rootIndex])
image1 = applyManipulation(st.image, bestSpans, bestNumSpans)
(newClass, newConfident) = model.predict(image1)
newClassStr = dataset.labels(int(newClass))
re = newClass != originalClass
path0 = "%s/%s_%s_modified_into_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imgIdx, origClassStr, newClassStr,
newConfident)
dataset.save(-1, image1, path0)
#print np.max(image1), np.min(image1)
print("difference between images: %s" %
(basics.diffImage(image, image1)))
#plt.imshow(image1 * 255, cmap=mpl.cm.Greys)
#plt.show()
if re == True:
eudist = basics.euclideanDistance(st.image, image1)
l1dist = basics.l1Distance(st.image, image1)
l0dist = basics.l0Distance(st.image, image1)
percent = basics.diffPercent(st.image, image1)
print "euclidean distance %s" % (eudist)
print "L1 distance %s" % (l1dist)
print "L0 distance %s" % (l0dist)
print "manipulated percentage distance %s" % (percent)
print "class is changed into %s with confidence %s\n" % (
newClassStr, newConfident)
dc.addEuclideanDistance(eudist)
dc.addl1Distance(l1dist)
dc.addl0Distance(l0dist)
dc.addManipulationPercentage(percent)
st.destructor()
else:
print("layer %s is of type %s, skipping" % (k, layerType))
#return
runningTime = time.time() - start_time
dc.addRunningTime(runningTime)
if re == True and cfg.exitWhen == "foundFirst":
break
k += 1
print("Please refer to the file %s for statistics." % (dc.fileName))
return re
def sampleNext(self, k):
#print("k=%s"%k)
#for j in self.keypoints:
# print(len(self.availableActionIDs[j]))
#print("oooooooo")
activations1 = applyManipulation(self.activations, self.spansPath,
self.numSpansPath)
(newClass, newConfident) = self.predictWithActivations(activations1)
(distMethod, distVal) = cfg.controlledSearch
if distMethod == "euclidean":
dist = basics.euclideanDistance(activations1, self.activations)
termValue = 0.0
termByDist = dist > distVal
elif distMethod == "L1":
dist = basics.l1Distance(activations1, self.activations)
termValue = 0.0
termByDist = dist > distVal
elif distMethod == "Percentage":
dist = basics.diffPercent(activations1, self.activations)
termValue = 0.0
termByDist = dist > distVal
elif distMethod == "NumDiffs":
dist = basics.diffPercent(activations1,
self.activations) * self.activations.size
termValue = 0.0
termByDist = dist > distVal
#if termByDist == False and newConfident < 0.5 and self.depth <= 3:
# termByDist = True
#self.re_training.addDatum(activations1,self.originalClass,newClass)
if newClass != self.originalClass and newConfident > effectiveConfidenceWhenChanging:
# and newClass == dataBasics.next_index(self.originalClass,self.originalClass):
basics.nprint(
"sampling a path ends in a terminal node with self.depth %s... "
% self.depth)
#print("L1 distance: %s"%(l1Distance(self.activations,activations1)))
#print(self.activations.shape)
#print(activations1.shape)
#print("L1 distance with KL: %s"%(withKL(l1Distance(self.activations,activations1),self.activations,activations1)))
(self.spansPath,
self.numSpansPath) = self.scrutinizePath(self.spansPath,
self.numSpansPath,
newClass)
#self.decisionTree.addOnePath(dist,self.spansPath,self.numSpansPath)
self.numAdv += 1
#self.analyseAdv.addAdv(activations1)
self.getUsefulPixels(self.accDims, self.d)
self.re_training.addDatum(activations1, self.originalClass,
newClass)
if self.bestCase[0] < dist:
self.numConverge += 1
self.bestCase = (dist, self.spansPath, self.numSpansPath)
path0 = "%s/%s_currentBest_%s_as_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, cfg.startIndexOfImage,
self.numConverge, self.dataset.labels(
int(newClass)), newConfident)
self.dataset.save(-1, activations1, path0)
return (self.depth == 0, dist)
elif termByDist == True:
basics.nprint(
"sampling a path ends by controlled search with self.depth %s ... "
% self.depth)
self.re_training.addDatum(activations1, self.originalClass,
newClass)
return (self.depth == 0, termValue)
elif list(
set(self.availableActionIDs[k]) -
set(self.usedActionIDs[k])) == []:
basics.nprint(
"sampling a path ends with self.depth %s because no more actions can be taken ... "
% self.depth)
return (self.depth == 0, termValue)
else:
#print("continue sampling node ... ")
#allChildren = initialisePixelSets(self.model,self.activations,self.spansPath.keys())
randomActionIndex = random.choice(
list(
set(self.availableActionIDs[k]) -
set(self.usedActionIDs[k]))
) #random.randint(0, len(allChildren)-1)
if k == 0:
span = {}
numSpan = {}
else:
(span, numSpan, _) = self.actions[k][randomActionIndex]
self.availableActionIDs[k].remove(randomActionIndex)
self.usedActionIDs[k].append(randomActionIndex)
newSpanPath = self.mergeSpan(self.spansPath, span)
newNumSpanPath = self.mergeNumSpan(self.numSpansPath, numSpan)
activations2 = applyManipulation(self.activations, newSpanPath,
newNumSpanPath)
(newClass2,
newConfident2) = self.predictWithActivations(activations2)
confGap2 = newConfident - newConfident2
if newClass2 == newClass:
self.accDims.append((randomActionIndex, confGap2))
else:
self.accDims.append((randomActionIndex, 1.0))
self.spansPath = newSpanPath
self.numSpansPath = newNumSpanPath
self.depth = self.depth + 1
self.accDims = self.accDims
self.d = self.d
if k == 0:
return self.sampleNext(randomActionIndex)
else:
return self.sampleNext(0)
def conv_safety_solve(layer2Consider,nfeatures,nfilters,filters,bias,input,activations,prevSpan,prevNumSpan,span,numSpan,pk):
random.seed(time.time())
# number of clauses
c = 0
# number of variables
d = 0
# variables to be used for z3
variable={}
if nfeatures == 1: images = np.expand_dims(input, axis=0)
else: images = input
if len(activations.shape) == 3:
avg = np.sum(activations)/float(len(activations)*len(activations[0])*len(activations[0][0]))
elif len(activations[0].shape) == 2:
avg = np.sum(activations)/float(len(activations)*len(activations[0]))
else: avg = 0
s = Tactic('qflra').solver()
s.reset()
#print("%s\n%s\n%s\n%s"%(prevSpan,prevNumSpan,span,numSpan))
toBeChanged = []
if inverseFunction == "point":
if nfeatures == 1:
#print("%s\n%s"%(nfeatures,prevSpan.keys()))
ks = [ (0,x,y) for (x,y) in prevSpan.keys() ]
else:
ks = copy.deepcopy(prevSpan.keys())
toBeChanged = toBeChanged + ks
elif inverseFunction == "area":
for (k,x,y) in span.keys():
toBeChanged = toBeChanged + [(l,x1,y1) for l in range(nfeatures) for x1 in range(x,x+cfg.filterSize) for y1 in range(y,y+cfg.filterSize) if x1 >= 0 and y1 >= 0 and x1 < images.shape[1] and y1 < images.shape[2]]
toBeChanged = list(set(toBeChanged))
for (l,x,y) in toBeChanged:
variable[1,0,l+1,x,y] = Real('1_x_%s_%s_%s' % (l+1,x,y))
d += 1
if not(cfg.boundOfPixelValue == [0,0]) and (layer2Consider == 0):
pstr = eval("variable[1,0,%s,%s,%s] <= %s"%(l+1,x,y,cfg.boundOfPixelValue[1]))
pstr = And(eval("variable[1,0,%s,%s,%s] >= %s"%(l+1,x,y,cfg.boundOfPixelValue[0])), pstr)
pstr = And(eval("variable[1,0,%s,%s,%s] != %s"%(l+1,x,y,images[l][x][y])), pstr)
s.add(pstr)
c += 1
maxterms = ""
for (k,x,y) in span.keys():
variable[1,1,k+1,x,y] = Real('1_y_%s_%s_%s' % (k+1,x,y))
d += 1
string = "variable[1,1,%s,%s,%s] == "%(k+1,x,y)
for l in range(nfeatures):
for x1 in range(cfg.filterSize):
for y1 in range(cfg.filterSize):
if (l,x+x1,y+y1) in toBeChanged:
newstr1 = " variable[1,0,%s,%s,%s] * %s + "%(l+1,x+x1,y+y1,filters[l,k][x1][y1])
elif x+x1 < images.shape[1] and y+y1 < images.shape[2] :
newstr1 = " %s + "%(images[l][x+x1][y+y1] * filters[l,k][x1][y1])
string += newstr1
string += str(bias[l,k])
s.add(eval(string))
c += 1
if cfg.enumerationMethod == "line":
pstr = eval("variable[1,1,%s,%s,%s] < %s" %(k+1,x,y,activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] + cfg.epsilon))
pstr = And(eval("variable[1,1,%s,%s,%s] > %s "%(k+1,x,y,activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] - cfg.epsilon)), pstr)
elif cfg.enumerationMethod == "convex" or cfg.enumerationMethod == "point":
if activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] >= 0:
upper = activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] + pk
lower = -1 * (activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)]) - pk
else:
upper = -1 * (activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)]) + pk
lower = activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] - pk
#if span[(k,x,y)] > 0 :
# upper = activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] + epsilon
# lower = activations[k][x][y] - span[(k,x,y)] * numSpan[(k,x,y)] - epsilon
#else:
# upper = activations[k][x][y] - span[(k,x,y)] * numSpan[(k,x,y)] + epsilon
# lower = activations[k][x][y] + span[(k,x,y)] * numSpan[(k,x,y)] - epsilon
#if span[(k,x,y)] > 0 :
# upper = activations[k][x][y] + span[(k,x,y)] + epsilon
# lower = activations[k][x][y] - epsilon - span[(k,x,y)]
#else:
# upper = activations[k][x][y] + epsilon - span[(k,x,y)]
# lower = activations[k][x][y] + span[(k,x,y)] - epsilon
pstr = eval("variable[1,1,%s,%s,%s] < %s"%(k+1,x,y,upper))
pstr = And(eval("variable[1,1,%s,%s,%s] > %s"%(k+1,x,y,lower)), pstr)
pstr = And(eval("variable[1,1,%s,%s,%s] != %s"%(k+1,x,y,activations[k][x][y])), pstr)
s.add(pstr)
c += 1
if activations[k][x][y] > 0 :
maxterm = "- variable[1,1,%s,%s,%s]"%(k+1,x,y)
else:
maxterm = "variable[1,1,%s,%s,%s]"%(k+1,x,y)
if maxterms == "":
maxterms = "(%s)"%maxterm
else:
maxterms = " %s + (%s) "%(maxterms, maxterm)
nprint("Number of variables: " + str(d))
nprint("Number of clauses: " + str(c))
p = multiprocessing.Process(target=s.check)
p.start()
# Wait for timeout seconds or until process finishes
p.join(cfg.timeout)
# If thread is still active
if p.is_alive():
print "Solver running more than timeout seconds (default="+str(cfg.timeout)+"s)! Skip it"
p.terminate()
p.join()
else:
s_return = s.check()
if 's_return' in locals():
if s_return == sat:
inputVars = [ (l,x,y,eval("variable[1,0,"+ str(l+1) +"," + str(x) +"," + str(y)+ "]")) for (l,x,y) in toBeChanged ]
cex = copy.deepcopy(images)
for (l,x,y,v) in inputVars:
#if cex[l][x][y] != v: print("different dimension spotted ... ")
cex[l][x][y] = getDecimalValue(s.model()[v])
#print("%s %s"%(images[l][x][y],cex[l][x][y]))
cex = np.squeeze(cex)
#cex2 = copy.deepcopy(activations)
#nextVars = [ (k,x,y,eval("variable[1,1,"+ str(k+1) +"," + str(x) +"," + str(y)+ "]")) for (k,x,y) in span.keys() ]
#for (k,x,y,v) in nextVars:
#if cex[l][x][y] != v: print("different dimension spotted ... ")
# cex2[k][x][y] = getDecimalValue(s.model()[v])
# print("%s %s"%(activations[k][x][y],cex2[k][x][y]))
nprint("satisfiable!")
return (True, cex)
else:
nprint("unsatisfiable!")
return (False, input)
else:
print "timeout! "
return (False, input)
def dense_safety_solve(nfeatures, nfilters, filters, bias, input, activations,
pcl, pgl, span, numSpan, pk):
random.seed(time.time())
rn = random.random()
# number of clauses
c = 0
# number of variables
d = 0
# variables to be used for z3
variable = {}
#print(filters)
#print(bias)
s = z3.Tactic('qflra').solver()
s.reset()
for l in pcl.keys():
variable[1, 0, l + 1] = z3.Real('1_x_%s' % (l + 1))
d += 1
for k in span.keys():
variable[1, 1, k + 1] = z3.Real('1_y_%s' % (k + 1))
d += 1
string = "variable[1,1,%s] == " % (k + 1)
for l in range(nfeatures):
if l in pcl.keys():
newstr1 = " variable[1,0,%s] * %s + " % (l + 1, filters[l, k])
else:
newstr1 = " %s + " % (input[l] * filters[l, k])
string += newstr1
string += str(bias[l, k])
s.add(eval(string))
#print(eval(string))
c += 1
pStr1 = "variable[1,1,%s] == %s" % (k + 1, activations[k])
s.add(eval(pStr1))
c += 1
basics.nprint("Number of variables: " + str(d))
basics.nprint("Number of clauses: " + str(c))
p = multiprocessing.Process(target=s.check)
p.start()
# Wait for timeout seconds or until process finishes
p.join(cfg.timeout)
# If thread is still active
if p.is_alive():
print "Solver running more than timeout seconds (default=" + str(
cfg.timeout) + "s)! Skip it"
p.terminate()
p.join()
else:
s_return = s.check()
if 's_return' in locals():
if s_return == z3.sat:
inputVars = [(l, eval("variable[1,0,%s]" % (l + 1)))
for l in pcl.keys()]
cex = copy.deepcopy(input)
for (l, x) in inputVars:
cex[l] = getDecimalValue(s.model()[x])
basics.nprint("satisfiable!")
return (True, cex)
else:
basics.nprint("unsatisfiable!")
return (False, input)
else:
print "unsatisfiable!"
return (False, input)
def safety_analysis(model, dataset, layer2Consider, imageIndex, st, index, cl2,
gl2, cp):
originalIndex = copy.deepcopy(index)
(originalImage, prevSpan, prevNumSpan, numDimsToMani,
stepsUpToNow) = st.getInfo(index)
config = model.getConfig()
# get weights and bias of the entire trained neural network
(wv, bv) = model.getWeightVector(layer2Consider)
# save the starting layer
originalLayer2Consider = copy.deepcopy(layer2Consider)
# predict with neural network
(originalSpanass, originalConfident) = model.predict(originalImage)
classstr = "the right class is " + (str(
dataset.labels(int(originalSpanass))))
print classstr
classstr = "the confidence is " + (str(originalConfident))
print classstr
print "safety analysis for layer %s ... " % (layer2Consider)
# wk is the set of inputs whose classes are different with the original one
wk = []
# wk is for the set of inputs need to be further considered
rk = [(originalImage, originalConfident)]
rkupdated = False
# rs remembers how many points have been tested in this round
rs = 0
# decide new span,numSpan according to precision cp
(span, numSpan) = (cl2, gl2)
#(span,numSpan) = decideNewP(cl2,gl2,cp)
#print "the numbers of spans are updated into %s ... "%(numSpan)
originalSpan = copy.deepcopy(span)
originalNumSpan = copy.deepcopy(numSpan)
if cfg.enumerationMethod == "convex":
allRounds = reduce(mul, map(lambda x: 2, numSpan.values()), 1)
elif cfg.enumerationMethod == "line":
allRounds = reduce(mul, map(lambda x: 2 * x + 1, numSpan.values()), 1)
elif cfg.enumerationMethod == "point":
allRounds = 1
print "%s regions need to be checked. " % (allRounds)
# counter_numSpan tracks the working point
# InitialisedNumSpan remembers
(counter_numSpan, InitialisedNumSpan) = initialiseCounter(numSpan)
#print("%s\n%s\n%s\n%s"%(counter_numSpan,InitialisedNumSpan,span,numSpan))
counter_numSpans = {}
counter_numSpans[originalLayer2Consider] = counter_numSpan
round = 0
cond = False
rn = 0
# a recursive procedure until find an interpolated image which is classified
# differently with the original image
# note: there are some other exit mechanisms in the looping body
while (not cond) or layer2Consider > 0:
if layer2Consider == originalLayer2Consider:
activations = model.getActivationValue(originalLayer2Consider,
originalImage)
activations1 = imageFromGL(activations, counter_numSpan, span)
cond = equalCounters(counter_numSpan, numSpan)
basics.nprint("\nin round: %s / %s" % (round, allRounds))
basics.nprint("layer: " + str(layer2Consider))
basics.nprint("counter_numSpan %s" % (counter_numSpan))
basics.nprint("numSpan %s" % (numSpan))
#print "maximal point %s"%(numSpan)
#print "activations1=%s"%(activations1)
# get the type of the current layer
layerType = model.getLayerType(layer2Consider)
#[ lt for (l,lt) in config if layer2Consider == l ]
#if len(layerType) > 0: layerType = layerType[0]
#else: print "cannot find the layerType"
# get the weights and bias for the current layer
wv2Consider, bv2Consider = basics.getWeight(wv, bv, layer2Consider)
# call different solving approaches according to
# the type of the layer and the type of the algorithm
# FIXME: need to expand this to work with other cases, e.g., MaxPooling2D, Convolution3D
if layerType == "Convolution2D" or layerType == "Conv2D":
basics.nprint("convolutional layer, back-propagating ...")
if layer2Consider == 0:
activations0 = copy.deepcopy(originalImage)
else:
activations0 = model.getActivationValue(
layer2Consider - 1, originalImage)
string = cfg.directory_pic_string + "/" + str(
imageIndex) + "_original_as_" + str(originalSpanass)
(bl, newInput) = conv_solve_prep(
model, cfg.dataBasics, string, originalLayer2Consider,
layer2Consider, prevSpan, prevNumSpan, counter_numSpan,
numSpan, cp, activations0, wv2Consider, bv2Consider,
activations1)
elif layerType == "Dense":
basics.nprint("dense layer, back propogation ... ")
if layer2Consider == 0:
activations0 = copy.deepcopy(originalImage)
else:
activations0 = model.getActivationValue(
layer2Consider - 1, originalImage)
string = cfg.directory_pic_string + "/" + str(
imageIndex) + "_original_as_" + str(originalSpanass)
(bl, newInput) = dense_solve_prep(model, cfg.dataBasics, string,
prevSpan, prevNumSpan,
counter_numSpan, numSpan, cp,
activations0, wv2Consider,
bv2Consider, activations1)
elif layerType == "InputLayer":
basics.nprint("inputLayer layer, back-propagating ... ")
(bl, newInput) = (True, copy.deepcopy(activations1))
elif layerType == "relu":
basics.nprint("relu layer, back-propagating ...")
(bl, newInput) = (True, copy.deepcopy(activations1))
elif layerType == "MaxPooling2D":
basics.nprint("MaxPooling2D layer, solving ... ")
if layer2Consider == 0:
activations0 = copy.deepcopy(originalImage)
else:
activations0 = model.getActivationValue(
layer2Consider - 1, originalImage)
image1 = maxpooling_safety_solve(activations0, activations1)
(bl, newInput) = (True, image1)
elif layerType == "Flatten":
basics.nprint("Flatten layer, solving ... ")
if layer2Consider == 0:
activations0 = copy.deepcopy(originalImage)
else:
activations0 = model.getActivationValue(
layer2Consider - 1, originalImage)
image1 = flatten_safety_solve(activations0, activations1)
(bl, newInput) = (True, image1)
# decide the next step according to the results from the solving
if bl == False:
# if back-propagation fails
basics.nprint("back-propagation or solving fails ... ")
if rkupdated == False:
#rk.append((newInput,originalConfident))
#print originalConfident, confident, rk[0][1]
if counter_numSpan == numSpan:
rkupdated = True
layer2Consider = copy.deepcopy(originalLayer2Consider)
index = copy.deepcopy(originalIndex)
(image, prevSpan, prevNumSpan, numDimsToMani,
stepsUpToNow) = st.getInfo(originalIndex)
counter_numSpan = counter_numSpans[originalLayer2Consider]
span = copy.deepcopy(originalSpan)
numSpan = copy.deepcopy(originalNumSpan)
(_, InitialisedNumSpan) = initialiseCounter(numSpan)
counter_numSpan = counterPlusOne(counter_numSpan, numSpan,
InitialisedNumSpan)
counter_numSpans[originalLayer2Consider] = copy.deepcopy(
counter_numSpan)
round += 1
elif layer2Consider > 0:
# still not yet reach the input layer
# continue back-propagating
layer2Consider -= 1
activations1 = copy.deepcopy(newInput)
index = st.parentIndexForIntermediateNode(index, layer2Consider)
basics.nprint("backtrack to index %s in layer %s" %
(index, layer2Consider))
activations = model.getActivationValue(layer2Consider,
originalImage)
counter_numSpan = getCounter(activations, newInput, prevSpan,
prevNumSpan)
span = copy.deepcopy(prevSpan)
numSpan = copy.deepcopy(prevNumSpan)
(image, prevSpan, prevNumSpan, numDimsToMani,
stepsUpToNow) = st.getInfo(index)
counter_numSpans[layer2Consider] = copy.deepcopy(counter_numSpan)
elif withinRegion(newInput, st) == True:
# reached the input layer
# and has to be within the region
# check to see if the new input is classified wronextNumSpany.
rs += 1
#print "reach input layer"
if dataset.name == "imageNet":
newInput = normalise(newInput)
basics.nprint("counter: %s" %
counter_numSpans[originalLayer2Consider])
basics.nprint("input: %s" % newInput)
(newClass, confident) = model.predict(newInput)
if dataset.name == "twoDcurve":
plt.plot([newInput[0]], [newInput[1]], 'g.')
basics.nprint("confident level: " + str(confident))
# Great! we found an image which has different class with the original image
if newClass != originalSpanass:
newClassStr = dataset.labels(int(newClass))
origClassStr = dataset.labels(int(originalSpanass))
classstr = "Class changed! from " + str(
origClassStr) + " into " + str(newClassStr)
print classstr
rk.append((newInput, confident))
path1 = "%s/%s_%s_modified_into_%s_with_confidence_%s.png" % (
cfg.directory_pic_string, imageIndex, origClassStr,
newClassStr, confident)
dataset.save(index[0], newInput, path1)
# add a point whose class is wrong
wk.append(newInput)
if cfg.exitWhen == "foundFirst": break
else:
#oldconf = rk[0][1]
#if (rk[0][1] == originalConfident):
# rk = [(newInput,confident)]
# diffImage(originalImage,newInput)
#elif confident < oldconf:
# rk = rk + [(newInput,confident)]
# diffImage(originalImage,newInput)
if rkupdated == False:
rk.append((newInput, confident))
#print originalConfident, confident, rk[0][1]
if counter_numSpan == numSpan:
rkupdated = True
layer2Consider = copy.deepcopy(originalLayer2Consider)
index = copy.deepcopy(originalIndex)
(image, prevSpan, prevNumSpan, numDimsToMani,
stepsUpToNow) = st.getInfo(originalIndex)
counter_numSpan = counter_numSpans[originalLayer2Consider]
span = copy.deepcopy(originalSpan)
numSpan = copy.deepcopy(originalNumSpan)
(_, InitialisedNumSpan) = initialiseCounter(numSpan)
counter_numSpan = counterPlusOne(counter_numSpan, numSpan,
InitialisedNumSpan)
counter_numSpans[originalLayer2Consider] = copy.deepcopy(
counter_numSpan)
round += 1
rn += 1
#path2 = directory_pic_string+"/temp%s_(round=%s).png"%(rn,round)
#path2 = directory_pic_string+"/temp.png"
#dataBasics.save(index[0],newInput, path2)
else:
rs += 1
if rkupdated == False:
#rk.append((newInput,originalConfident))
#print originalConfident, confident, rk[0][1]
if counter_numSpan == numSpan:
rkupdated = True
layer2Consider = copy.deepcopy(originalLayer2Consider)
index = copy.deepcopy(originalIndex)
(image, prevSpan, prevNumSpan, numDimsToMani,
stepsUpToNow) = st.getInfo(originalIndex)
counter_numSpan = counter_numSpans[originalLayer2Consider]
span = copy.deepcopy(originalSpan)
numSpan = copy.deepcopy(originalNumSpan)
(_, InitialisedNumSpan) = initialiseCounter(numSpan)
counter_numSpan = counterPlusOne(counter_numSpan, numSpan,
InitialisedNumSpan)
counter_numSpans[originalLayer2Consider] = copy.deepcopy(
counter_numSpan)
round += 1
#print("2%s---%s"%(rkupdated,rk))
print("ran througn the neural network for %s times." % (rn))
#path2 = directory_pic_string+"/temp%s_%s.png"%(howfar,originalLayer2Consider)
#dataBasics.save(newInput, path2)
rk = rk[1:]
if rk != []:
rk.sort(key=lambda x: -1 * x[1])
return (span, numSpan, rs, wk, rk)