def test_execute_extracts_visibility_comparison_and_concatenate_other_features(
self):
arff_data = arff.load(
self.generate_arff("""13,17,1,2,3,4,5,6,7,8,100,0.12,360,414,0.3,0
0,0,100,150,20,15,10,15,20,33,1000,0.25,360,380,0.15,0""",
extractors=[
ComplexityExtractor(),
ImageComparisonExtractor(),
SizeViewportExtractor(),
VisibilityExtractor()
]))
arff_data['data'] = np.array(arff_data['data'])
arff_data['X'] = np.array([[1, 7], [7, 7]])
arff_data['features'] = ['super', 'bash']
result = self.extractor.execute(arff_data)
# right = (354 - 360) - (406 - 414)
self.assertEqual(1, result['X'][0][0])
self.assertEqual(7, result['X'][0][1])
self.assertEqual(2, result['X'][0][10])
self.assertEqual(53, result['X'][0][11])
# right = (240 - 360) - (197 - 380)
self.assertEqual(7, result['X'][1][0])
self.assertEqual(7, result['X'][1][1])
self.assertEqual(63, result['X'][1][10])
self.assertEqual(-30, result['X'][1][11])
self.assertEqual([
'super', 'bash', 'childsNumber', 'textLength', 'area', 'phash',
'chiSquared', 'imageDiff', 'width_comp', 'height_comp',
'left_visibility', 'right_visibility'
], result['features'])
def execute(self, dataset):
print('===== Feature selection - Embeddings Clustering =====')
texts = [text_data['content'] for text_data in dataset]
self._vectorizer.fit(texts)
#if (len(self._vectorizer.vocabulary_) < self._k):
# print('Number of unique words is smaller than number of clusters (%d < %d)' %
# (len(self._vectorizer.vocabulary_), self._k))
# return dataset
self._embedding_matrix = self._loader.load(self._vectorizer)
print('===== K-Means %d =====' % (self._k))
model = KMeans(n_clusters=self._k, random_state=self._random_state)
np_embedding_matrix = np.array(list(self._embedding_matrix.values()))
model.fit(np_embedding_matrix)
print('===== replacing similar words by similarity =====')
for text_data in dataset:
tokens = word_tokenize(text_data['content'])
new_tokens = []
for token in tokens:
try:
word_embedding = self._embedding_matrix[token]
word_cluster = model.predict(np.array([word_embedding]))[0]
new_tokens.append('token' + str(word_cluster))
except:
#print('Key not found in index, removing word...')
pass
text_data['content'] = ' '.join(new_tokens)
return dataset
def plotData(trainC, trainF, trainO, coef, name):
noOfPoints = 1000
xref = []
val = min(trainC)
step = (max(trainC) - min(trainF)) / noOfPoints
xref1 = []
val1 = min(trainF)
step1 = (max(trainF) - min((trainF))) / noOfPoints
for i in range(1, noOfPoints):
xref.append(val)
xref1.append(val1)
val += step
val1 += step1
yref = [[coef[0] + coef[1] * x1 + coef[2] * y1]
for x1, y1 in zip(xref, xref1)]
ax = plt.axes(projection='3d')
ax.scatter3D(trainC, trainF, trainO, c='b', marker='o')
ax.plot_surface(np.array(xref),
np.array(xref1),
np.array(yref),
color='green')
ax.set_xlabel('Capital')
ax.set_ylabel('Freedom')
ax.set_zlabel('Happiness')
plt.title(name)
plt.show()
def execute(self, arff_data, attributes, X):
base_fonts = arff_data['data'][:, attributes.index('baseFontFamily')]
one_hot = OneHotEncoder(handle_unknown='ignore', sparse=False)
label = LabelEncoder()
label_encoded = label.fit_transform(base_fonts)
encoded_base_fonts = one_hot.fit_transform(label_encoded.reshape(
-1, 1))
target_fonts = arff_data['data'][:,
attributes.index('targetFontFamily')]
enc = OneHotEncoder(handle_unknown='ignore', sparse=False)
label = LabelEncoder()
label_encoded = label.fit_transform(target_fonts)
encoded_target_fonts = enc.fit_transform(label_encoded.reshape(-1, 1))
X_list = X
if (encoded_base_fonts.shape[1] == 1):
encoded_base_fonts = [encoded_base_fonts.reshape(-1).tolist()]
else:
encoded_base_fonts = np.array(encoded_base_fonts).T.tolist()
for font_column in encoded_base_fonts:
X_list.append(font_column)
if (encoded_target_fonts.shape[1] == 1):
encoded_target_fonts = [encoded_target_fonts.reshape(-1).tolist()]
else:
encoded_target_fonts = np.array(encoded_target_fonts).T.tolist()
for font_column in encoded_target_fonts:
X_list.append(font_column)
return X_list
def test_extractor_implements_execute_method_with_correct_arguments(self):
X = np.array([[1, 2, 3], [11, 12, 13]])
arff_string = """1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,1
1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,1"""
(arff_data, attributes) = self.generate_arff(arff_string)
result = self.extractor.execute(arff_data, attributes, X.T.tolist())
result = np.array(result).T.tolist()
self.assertEqual(2, len(result))
self.assertEqual(X.tolist(), (np.array(result)[0:2, 0:3]).tolist())
def execute(self, arff_data, attributes, X):
X.append(arff_data['data'][:, attributes.index('childsNumber')])
X.append(arff_data['data'][:, attributes.index('textLength')])
base_width = np.array(arff_data['data'][:, attributes.index('baseWidth')], dtype='float64')
target_width = np.array(arff_data['data'][:, attributes.index('targetWidth')], dtype='float64')
base_height = np.array(arff_data['data'][:, attributes.index('baseHeight')], dtype='float64')
target_height = np.array(arff_data['data'][:, attributes.index('targetHeight')], dtype='float64')
X.append(np.maximum(base_width * base_height, target_width * target_height))
arff_data['features'] = arff_data['features'] + ['childsNumber', 'textLength', 'area']
return X
def execute(self, arff_data):
arff_data['features'] = self._features
attributes = [attribute[0] for attribute in arff_data['attributes']]
Xt = []
for feature in self._features:
Xt.append(arff_data['data'][:, attributes.index(feature)])
arff_data['X'] = np.array(Xt, dtype='float64').T
arff_data['y'] = np.array(
arff_data['data'][:, attributes.index(self._class_label)])
return arff_data
def read(startExample, count, digits=None, bTrain=True, path="."):
if digits == None:
digits = range(0, 10)
if bTrain:
fname_img = os.path.join(path, 'train-images-idx3-ubyte')
fname_lbl = os.path.join(path, 'train-labels-idx1-ubyte')
else:
fname_img = os.path.join(path, 't10k-images-idx3-ubyte')
fname_lbl = os.path.join(path, 't10k-labels-idx1-ubyte')
fImages = open(fname_img,'rb')
fLabels = open(fname_lbl,'rb')
# read the header information in the images file.
s1, s2, s3, s4 = fImages.read(4), fImages.read(4), fImages.read(4), fImages.read(4)
mnIm = struct.unpack('>I',s1)[0]
numIm = struct.unpack('>I',s2)[0]
rowsIm = struct.unpack('>I',s3)[0]
colsIm = struct.unpack('>I',s4)[0]
# seek to the image we want to start on
fImages.seek(16+startExample*rowsIm*colsIm)
# read the header information in the labels file and seek to position
# in the file for the image we want to start on.
mnL = struct.unpack('>I',fLabels.read(4))[0]
numL = struct.unpack('>I',fLabels.read(4))[0]
fLabels.seek(8+startExample)
inputVectors = [] # list of (input, correct label) pairs
labels = []
for c in range(count):
# get the correct label from the labels file.
val = struct.unpack('>B',fLabels.read(1))[0]
labels.append(val)
vec = map(lambda x: struct.unpack('>B',fImages.read(1))[0],
range(rowsIm*colsIm))
# get the input from the image file
inputVectors.append(np.array(vec))
# Filter out the unwanted digits
ind = [k for k in xrange(len(labels)) if labels[k] in digits ]
labels = map(lambda x: labels[x], ind)
inputVectors = map(lambda x: inputVectors[x], ind)
fImages.close()
fLabels.close()
return np.array(inputVectors), np.array(labels)
def _divCross(self, other):
"""
|u x v| = |u||v|sin(theta)
"""
A = np.array([[0, -other.z, -other.y], [other.z, 0, other.x],
[-other.y, other.x, 0]])
if not det(A):
return SpatialVector()
b = np.array(self.vec)
return SpatialVector(*A.inv().dot(b))
def execute(self, arff_data, attributes, X):
X.append(arff_data['data'][:, attributes.index('phash')])
X.append(arff_data['data'][:, attributes.index('chiSquared')])
image_diff = np.array(arff_data['data'][:, attributes.index('imageDiff')], dtype='float64')
base_width = np.array(arff_data['data'][:, attributes.index('baseWidth')], dtype='float64')
target_width = np.array(arff_data['data'][:, attributes.index('targetWidth')], dtype='float64')
base_height = np.array(arff_data['data'][:, attributes.index('baseHeight')], dtype='float64')
target_height = np.array(arff_data['data'][:, attributes.index('targetHeight')], dtype='float64')
min_area = np.minimum(base_width * base_height, target_width * target_height)
X.append(image_diff / (np.maximum(min_area, 1) * 255))
arff_data['features'] = arff_data['features'] + ['phash', 'chiSquared', 'imageDiff']
return X
def hsv(self, path='./photo/4.jpg'):
# icol = (36,202,59,71,255,255) # Green
#icol = (18, 0, 196, 36, 255, 255) # Yellow
#icol = (89, 0, 0, 125, 255, 255) # Blue
icol = (0, 100, 80, 10, 255, 255) # Red
cv.namedWindow('colorTest')
# Lower range colour sliders.
cv.createTrackbar('lowHue', 'colorTest', icol[0], 255, self.nothing)
cv.createTrackbar('lowSat', 'colorTest', icol[1], 255, self.nothing)
cv.createTrackbar('lowVal', 'colorTest', icol[2], 255, self.nothing)
# Higher range colour sliders.
cv.createTrackbar('highHue', 'colorTest', icol[3], 255, self.nothing)
cv.createTrackbar('highSat', 'colorTest', icol[4], 255, self.nothing)
cv.createTrackbar('highVal', 'colorTest', icol[5], 255, self.nothing)
frame = cv.imread(path)
while True:
lowHue = cv.getTrackbarPos('lowHue', 'colorTest')
lowSat = cv.getTrackbarPos('lowSat', 'colorTest')
lowVal = cv.getTrackbarPos('lowVal', 'colorTest')
highHue = cv.getTrackbarPos('highHue', 'colorTest')
highSat = cv.getTrackbarPos('highSat', 'colorTest')
highVal = cv.getTrackbarPos('highVal', 'colorTest')
cv.imshow('frame', frame)
frameBGR = cv.GaussianBlur(frame, (7, 7), 0)
cv.imshow('blurred', frameBGR)
hsv = cv.cvtColor(frameBGR, cv.COLOR_BGR2HSV)
# HSV values to define a colour range.
colorLow = np.array([lowHue, lowSat, lowVal])
colorHigh = np.array([highHue, highSat, highVal])
mask = cv.inRange(hsv, colorLow, colorHigh)
# Show the first mask
cv.imshow('mask-plain', mask)
kernal = cv.getStructuringElement(cv.MORPH_ELLIPSE, (7, 7))
mask = cv.morphologyEx(mask, cv.MORPH_CLOSE, kernal)
mask = cv.morphologyEx(mask, cv.MORPH_OPEN, kernal)
cv.imshow('mask', mask)
result = cv.bitwise_and(frame, frame, mask=mask)
cv.imshow('colorTest', result)
k = cv.waitKey(5) & 0xFF
if k == 27:
break
def test_calling_tunning_methods(self):
grid_mock = Mock()
grid_mock.best_params_ = {'params': True}
data = {
'X': np.array([[1, 2, 3], [4, 5, 6]]),
'y': np.array(['none', 'xbi']),
}
model = Mock()
action = ClassifierTunning(grid_mock, model)
result = action.execute(data)
grid_mock.fit.assert_called_with(data['X'], data['y'])
model.set_params.assert_called_with(**grid_mock.best_params_)
self.assertEqual(model, result['model'])
def execute(self, arff_data):
attributes = [attribute[0] for attribute in arff_data['attributes']]
dataset = arff_data['data']
X = (arff_data['X'].T.tolist() if 'X' in arff_data else [])
base_height = np.array(dataset[:, attributes.index('baseHeight')],
dtype='float64')
target_height = np.array(dataset[:,
attributes.index('targetHeight')],
dtype='float64')
base_width = np.array(dataset[:, attributes.index('baseWidth')],
dtype='float64')
target_width = np.array(dataset[:, attributes.index('targetWidth')],
dtype='float64')
X.append(
np.minimum(base_height * base_width, target_height * target_width))
base_x = np.array(dataset[:, attributes.index('baseX')],
dtype='float64')
target_x = np.array(dataset[:, attributes.index('targetX')],
dtype='float64')
base_y = np.array(dataset[:, attributes.index('baseY')],
dtype='float64')
target_y = np.array(dataset[:, attributes.index('targetY')],
dtype='float64')
X.append(
np.sqrt(
np.power(np.abs(base_x - target_x), 2) +
np.power(np.abs(base_y - target_y), 2)))
X.append(
np.abs((base_height * base_width) -
(target_height * target_width)) / np.maximum(
np.minimum(base_height * base_width, target_height *
target_width), np.ones(len(base_height))))
X.append(dataset[:, attributes.index('chiSquared')])
arff_data['X'] = np.array(X, dtype='float64').T
prev_features = (arff_data['features']
if 'features' in arff_data else [])
arff_data['features'] = prev_features + [
'area', 'displacement', 'sdr', 'chisquared'
]
arff_data['y'] = np.array(
arff_data['data'][:, attributes.index(self._class_attr)],
dtype='int16')
return arff_data
def execute(self, arff_data):
arff_data['features'] = (arff_data['features'] if 'features' in arff_data else [])
attributes = [ attribute[0]
for attribute in arff_data['attributes'] ]
X = (arff_data['X'].T.tolist() if 'X' in arff_data else [])
for extractor in self._extractors:
X = extractor.execute(arff_data, attributes, X)
arff_data['X'] = np.array(X, dtype='float64').T
arff_data['y'] = np.array(arff_data['data'][:, attributes.index(self._class_attr)], dtype='float64')
return arff_data
def waitcontours():
while(1) :
ret, frame = cap.read()
gray = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) # name of the variable
lower = np.array([160])
upper = np.array([255])
# smooth the image thresholded
mask = cv.blur(cv.inRange(gray, lower, upper), ksize=(3, 3))
contours, hierarchy = cv.findContours(mask, cv.RETR_LIST, cv.CHAIN_APPROX_NONE)
if contours:
break
def test_real_dataset_for_measuring_X_matrix_conformance(self):
arff_data = self.generate_arff("""1,2,3,Arial,Bonito,1
7,8,9,Deustrech,Arial,1
10,11,12,Bonito,Arial,1
4,5,6,Bonito,Deustrech,1""")
X = np.ones(np.array(arff_data['data']).shape)
y = np.ones(X.shape[0])
arguments = {
'X': X,
'y': y,
'attributes': arff_data['attributes'],
'data': np.array(arff_data['data'])
}
result = self.extractor.execute(arguments)
self.assertEqual(y.shape[0], result['X'].shape[0])
def get_metric_value(self, stablization_window_size, ipList):
metrics_per_network = []
for ip in range(0, len(ipList)):
prometheus_url = ipList[ip]
# prometheus_query_url = "http://{}:19999/api/v1/allmetrics?format=prometheus".format(prometheus_url)
prometheus_query_url = "http://{}:19999/api/v1/data?chart=transit_xdp.pps&after=-1".format(
prometheus_url)
response = requests.get(prometheus_query_url).json()
# print(response['data'][0][1])
metric_value = response['data'][0][
1] # the transit_xdp.pps metrics
'''Write the data to the metric queue'''
poll_interval_seconds = self.config['poll_interval_seconds']
# print(math.ceil(stablization_window_size/poll_interval_seconds))
if len(self.metric_queue) > math.ceil(
stablization_window_size / poll_interval_seconds):
self.deQueue()
self.enQueue(metric_value)
else:
self.enQueue(metric_value)
metric_matrix = np.array([self.metric_queue])
# print("metrics Queue: {}".format(self.metric_queue))
# print("len(metric_matrix): {}".format(len(metric_matrix)))
# print("metric_matrix: {}".format(metric_matrix))
''' Calculate the average value (per bouncer) in the Window '''
''' Append the "per bouncer" metrics to "per network" metrics '''
metrics_per_network.append(
metric_matrix.mean(axis=1)) # Mean by row
# print("metrics_per_net: {}".format(metrics_per_network))
# print("metrics_per_net: {}".format(metrics_per_network[0]))
return metrics_per_network[0]
def separateOtsu(picture):
hist_count = np.array([i for i in range(GRAYLEVEL)])
for x in range(len(picture)):
for y in range(len(picture[0])):
grayscale = picture[x][y]
hist_count[grayscale] += 1
return hist_count
def img_kernel(self, img):
# kernel=np.ones([5,5],np.float32)/25
# kernel=np.array([[0,1,0],[0,1,0],[0,1,0]],np.float32)/3
#过滤片
kernel = np.array([[1, 0, -1], [1, 0, -1], [1, 0, -1]], np.float32)
dst = cv.filter2D(img, -1, kernel)
return dst
def watermark(self, src_path, mask_path, alpha=0.3):
img = cv.imread(src_path)
h, w = img.shape[0], img.shape[1]
mask = cv.imread(mask_path, cv.IMREAD_UNCHANGED)
if w > h:
rate = int(w * 0.1) / mask.shape[1]
else:
rate = int(h * 0.1) / mask.shape[0]
mask = cv.resize(mask, None, fx=rate, fy=rate)
mask_h, mask_w = mask.shape[0], mask.shape[1]
mask_channels = cv.split(mask)
dst_channels = cv.split(img)
b, g, r, a = cv.split(mask)
# 计算mask在图片的坐标
ul_points = (int(h * 0.9), int(int(w / 2) - mask_w / 2))
dr_points = (int(h * 0.9) + mask_h, int(int(w / 2) + mask_w / 2))
for i in range(3):
dst_channels[i][ul_points[0]:dr_points[0],
ul_points[1]:dr_points[1]] = dst_channels[i][
ul_points[0]:dr_points[0],
ul_points[1]:dr_points[1]] * (255.0 -
a * alpha) / 255
dst_channels[i][ul_points[0]:dr_points[0],
ul_points[1]:dr_points[1]] += np.array(
mask_channels[i] * (a * alpha / 255),
dtype=np.uint8)
dst_img = cv.merge(dst_channels)
self.show_img(dst_img)
def select_rgb_blue_black(image):
# define range of blue color in HSV
lower_blue = np.array([110, 50, 50])
upper_blue = np.array([130, 255, 255])
blue_mask = cv2.inRange(image, lower_blue, upper_blue)
# define range of black color in HSV
lower_black = np.array([0, 0, 0])
upper_black = np.array([170, 100, 50])
# Threshold the HSV image to get only blue colors
black_mask = cv2.inRange(image, lower_black, upper_black)
# Bitwise-AND mask and original image
mask = cv2.bitwise_or(blue_mask, black_mask)
res = cv2.bitwise_and(image, image, mask=blue_mask)
return mask
def execute(self, arff_data, attributes, X):
base_width = np.array(arff_data['data'][:, attributes.index('baseWidth')], dtype='float64')
target_width = np.array(arff_data['data'][:, attributes.index('targetWidth')], dtype='float64')
base_viewport = np.array(arff_data['data'][:, attributes.index('baseViewportWidth')], dtype='float64')
target_viewport = np.array(arff_data['data'][:, attributes.index('targetViewportWidth')], dtype='float64')
base_left = np.array(arff_data['data'][:, attributes.index('baseX')], dtype='float64')
base_right = np.array(base_viewport - (base_left + base_width), dtype='float64')
target_left = np.array(arff_data['data'][:, attributes.index('targetX')], dtype='float64')
target_right = np.array(target_viewport - (target_left + target_width), dtype='float64')
X.append(np.abs((base_left - target_left) / np.maximum(np.abs(base_viewport - target_viewport), 1)))
X.append(np.abs((base_right - target_right) / np.maximum(np.abs(base_viewport - target_viewport), 1)))
base_y = np.array(arff_data['data'][:, attributes.index('baseY')], dtype='float64')
target_y = np.array(arff_data['data'][:, attributes.index('targetY')], dtype='float64')
X.append(np.abs((base_y - target_y) / np.maximum(np.abs(base_viewport - target_viewport), 1)))
arff_data['features'] = arff_data['features'] + ['left_comp', 'right_comp', 'y_comp']
return X
def q_and_probs_from_engine_score(infos, probs, num_nodes, board, next_move_in_game):
boosting_nodes = math.ceil((num_nodes / 0.7) - num_nodes)
q = None
total_visited_nodes = 0
move_nodes = {}
for i, info in enumerate(infos):
if i == 0:
q = info["score"].relative.score(mate_score=100) / 10000
pythonchess_move = info['pv'][0]
move = unclean_uci_move_to_lc0(pythonchess_move.uci(), board)
node_count = info['nodes']
total_visited_nodes += node_count
move_nodes[move] = node_count
if pythonchess_move == next_move_in_game:
move_nodes[move] += boosting_nodes
total_visited_nodes += boosting_nodes
if total_visited_nodes == 0:
raise Exception("somehow no moves visited in position, crashing")
# create writeable probability array
probs = np.array(probs)
for move, node_count in move_nodes.items():
probs[constants.MOVES_LOOKUP[move]] = node_count / total_visited_nodes
return q, probs
def test_execute_extracts_platform_ids_with_position_viewport(self):
arff_data = arff.load("""@RELATION browserninja.website
@ATTRIBUTE childsNumber NUMERIC
@ATTRIBUTE textLength NUMERIC
@ATTRIBUTE baseX NUMERIC
@ATTRIBUTE targetX NUMERIC
@ATTRIBUTE baseY NUMERIC
@ATTRIBUTE targetY NUMERIC
@ATTRIBUTE baseHeight NUMERIC
@ATTRIBUTE targetHeight NUMERIC
@ATTRIBUTE baseWidth NUMERIC
@ATTRIBUTE targetWidth NUMERIC
@ATTRIBUTE imageDiff NUMERIC
@ATTRIBUTE chiSquared NUMERIC
@ATTRIBUTE baseViewportWidth NUMERIC
@ATTRIBUTE targetViewportWidth NUMERIC
@ATTRIBUTE phash NUMERIC
@ATTRIBUTE basePlatform STRING
@ATTRIBUTE targetPlatform STRING
@ATTRIBUTE Result {0,1}
@DATA
13,17,1,2,3,4,5,6,7,8,100,0.12,360,360,0.3,'iOS 12.1 - Safari -- iOS - iPhone 8','iOS 12.1 - Safari -- iOS - iPhoneSE',1
0,0,100,150,20,15,10,15,20,33,1000,0.25,360,360,0.15,'iOS 12.1 - Safari -- iOS - iPhone 8','iOS 12.1 - Safari -- iOS - iPhone 8 Plus',1
""")
arff_data['data'] = np.array(arff_data['data'])
self.extractor = BrowserNinjaCompositeExtractor(
class_attr='Result',
extractors=[PlatformExtractor(),
PositionViewportExtractor()])
result = self.extractor.execute(arff_data)
self.assertEqual(1, result['X'][0][0])
self.assertEqual(0, result['X'][1][0])
self.assertEqual(['platform_id', 'left_comp', 'right_comp', 'y_comp'],
result['features'])
def test_execute_comparison_based_on_viewport_when_zero_happens(self):
arff_data = arff.load(
self.generate_arff("""13,17,1,2,3,4,5,6,7,8,100,0.12,360,360,0.3,0
0,0,100,150,20,15,10,15,20,33,1000,0.25,360,360,0.15,0""",
extractors=[
ComplexityExtractor(),
ImageComparisonExtractor(),
SizeViewportExtractor(),
VisibilityExtractor(),
PositionViewportExtractor()
]))
arff_data['data'] = np.array(arff_data['data'])
result = self.extractor.execute(arff_data)
self.assertEqual(1, result['X'][0][10])
self.assertEqual(2, result['X'][0][11])
self.assertEqual(1, result['X'][0][12])
self.assertEqual(50, result['X'][1][10])
# 240 - 177
self.assertEqual(63, result['X'][1][11])
self.assertEqual(5, result['X'][1][12])
self.assertEqual([
'childsNumber', 'textLength', 'area', 'phash', 'chiSquared',
'imageDiff', 'width_comp', 'height_comp', 'left_visibility',
'right_visibility', 'left_comp', 'right_comp', 'y_comp'
], result['features'])
def dh(theta, d, a, alpha):
# First row of dh table
A11 = np.cos(theta)
A12 = -np.cos(alpha) * np.sin(theta)
A13 = np.sin(alpha) * np.sin(theta)
A14 = a * np.cos(theta)
# second row of dh table
A21 = np.sin(theta)
A22 = np.cos(alpha) * np.cos(theta)
A23 = -np.sin(alpha) * np.sin(theta)
A24 = a * np.sin(theta)
# third row of dh table
A31 = 0
A32 = np.sin(alpha)
A33 = np.cos(alpha)
A34 = d
# fourth row of dh table
A41 = 0
A42 = 0
A43 = 0
A44 = 1
A = 0
A = np.array([[A11, A12, A13, A14], [A21, A22, A23, A24],
[A31, A32, A33, A34], [A41, A42, A43, A44]])
return A
def detect(self, data):
try:
frame = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
frame = imutils.resize(frame, width=400)
cv2.imshow("Frame", frame)
# grab the frame dimensions and convert it to a blob
(h, w) = frame.shape[:2]
blob = cv2.dnn.blobFromImage(cv2.resize(frame, (224, 224)), 0.007843,
(224, 224), 127.5)
# pass the blob through the network and obtain the detections and
# predictions
self.net.setInput(blob)
detections = self.net.forward()
rois = []
# loop over the detections
for i in np.arange(0, detections.shape[2]):
# extract the confidence (i.e., probability) associated with
# the prediction
confidence = detections[0, 0, i, 2]
# filter out weak detections by ensuring the `confidence` is
# greater than the minimum confidence
if confidence > self.args["confidence"]:
# extract the index of the class label from the
# `detections`, then compute the (x, y)-coordinates of
# the bounding box for the object
idx = int(detections[0, 0, i, 1])
box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
(startX, startY, endX, endY) = box.astype("int")
# draw the prediction on the frame
label = "{}: {:.2f}%".format(self.CLASSES[idx],
confidence * 100)
cv2.rectangle(frame, (startX, startY), (endX, endY),
self.COLORS[idx], 2)
y = startY - 15 if startY - 15 > 15 else startY + 15
cv2.putText(frame, label, (startX, y),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.COLORS[idx], 2)
if self.CLASSES[idx] is "person":
rois.append((startX, startY, endX, endY))
self.pub_roi(rois)
# show the output frame
cv2.imshow("Frame", frame)
#key = cv2.waitKey(1) & 0xFF
cv2.waitKey(3)
#Only publish if we've detected a person
#if (len(pick) > 0):
# try:
self.image_pub.publish(self.bridge.cv2_to_imgmsg(frame, "bgr8"))
def test_extracts_features_from_different_data(self):
arff_example = """@RELATION otherdata
@ATTRIBUTE supimpa NUMERIC
@ATTRIBUTE nothing NUMERIC
@ATTRIBUTE anotherr NUMERIC
@ATTRIBUTE somethingelse NUMERIC
@ATTRIBUTE anotherthing NUMERIC
@ATTRIBUTE result {xbi, none}
@DATA
1,2,3,4,5,none
6,7,8,9,10,none
11,12,13,14,15,none
16,17,18,19,20,none
21,22,23,24,25,none
26,27,28,29,30,xbi"""
data = arff.load(arff_example)
data['data'] = np.array(data['data'])
extractor = XBIExtractor(['supimpa', 'somethingelse', 'nothing'],
'result')
result = extractor.execute(data)
self.assertEqual(['supimpa', 'somethingelse', 'nothing'],
result['features'])
self.assertEqual(6, len(result['X']))
self.assertEqual(3, len(result['X'][3]))
self.assertEqual(21.0, result['X'][4][0])
self.assertEqual(24.0, result['X'][4][1])
self.assertEqual(22.0, result['X'][4][2])
self.assertEqual(6, len(result['y']))
self.assertEqual('none', result['y'][0])
self.assertEqual('xbi', result['y'][5])
def load(self, vectorizer):
print('===== SE Embeddings loading from %s =====' %
(self._gensim_file))
self._se_embeddings = gensim.models.KeyedVectors.load_word2vec_format(
self._gensim_file, binary=True)
embedding_dim = self._embedding_dim
self._vocab_size = len(
vectorizer.vocabulary_
) + 1 # Adding again 1 because of reserved 0 index
#self._embedding_matrix = np.zeros((self._vocab_size, embedding_dim))
self._embedding_matrix = {}
not_found = []
for word in vectorizer.vocabulary_.keys():
try:
#idx = vectorizer.vocabulary_[word]
#self._embedding_matrix[idx] = np.array(
# self._se_embeddings.get_vector(word), dtype=np.float32)[:embedding_dim]
self._embedding_matrix[word] = np.array(
self._se_embeddings.get_vector(word),
dtype=np.float32)[:embedding_dim]
except:
not_found.append(word)
print('Not in embedding: %s...' % (not_found))
return self._embedding_matrix
def interPolateFunc(data): #data:n*3
try:
points = pandas.DataFrame(np.array(data)) #构造DataFame
points.set_axis(['lon', 'lat', 'depth'], axis='columns',
inplace=True) #重新设置列索引名称
grid_x, grid_y = np.mgrid[min(points.iloc[:,
0]):max(points.iloc[:,
0]):500j,
min(points.iloc[:,
1]):max(points.iloc[:,
1]):500j]
values = points.iloc[:, 2]
points2 = points.drop('depth', 1) #去掉最后一列
grid_z = griddata(points2, values, (grid_x, grid_y),
method='cubic') #插值nearest,linear,cubic
len_lon = len(grid_x) #经度数据点数
len_lat = len(grid_x[0]) #纬度数据点数
depth = grid_z.tolist() #深度数据
lon_start = min(points.iloc[:, 0]) #经度起点
lon_end = max(points.iloc[:, 0]) #经度终点
lat_start = min(points.iloc[:, 1]) #纬度起点
lat_end = max(points.iloc[:, 1]) #纬度终点
except BaseException:
len_lon = -1
len_lat = -1
return len_lon, len_lat
else:
return len_lon, len_lat, depth, lon_start, lon_end, lat_start, lat_end
def generate_arff(self, data, class_attr='Result'):
arff_header = """@RELATION browserninja.website
@ATTRIBUTE baseX NUMERIC
@ATTRIBUTE targetX NUMERIC
@ATTRIBUTE baseY NUMERIC
@ATTRIBUTE targetY NUMERIC
@ATTRIBUTE baseHeight NUMERIC
@ATTRIBUTE targetHeight NUMERIC
@ATTRIBUTE baseWidth NUMERIC
@ATTRIBUTE targetWidth NUMERIC
@ATTRIBUTE baseParentX NUMERIC
@ATTRIBUTE targetParentX NUMERIC
@ATTRIBUTE baseParentY NUMERIC
@ATTRIBUTE targetParentY NUMERIC
@ATTRIBUTE baseViewportWidth NUMERIC
@ATTRIBUTE targetViewportWidth NUMERIC
@ATTRIBUTE basePreviousSiblingLeft NUMERIC
@ATTRIBUTE targetPreviousSiblingLeft NUMERIC
@ATTRIBUTE basePreviousSiblingTop NUMERIC
@ATTRIBUTE targetPreviousSiblingTop NUMERIC
@ATTRIBUTE baseNextSiblingLeft NUMERIC
@ATTRIBUTE targetNextSiblingLeft NUMERIC
@ATTRIBUTE baseNextSiblingTop NUMERIC
@ATTRIBUTE targetNextSiblingTop NUMERIC
@ATTRIBUTE childsNumber NUMERIC
@ATTRIBUTE textLength NUMERIC
@ATTRIBUTE %s {0,1}
@DATA
""" % (class_attr)
arff_data = arff.load(arff_header + data)
attributes = [attribute[0] for attribute in arff_data['attributes']]
arff_data['data'] = np.array(arff_data['data'])
arff_data['features'] = []
return (arff_data, attributes)
def test_calculate_single_features_and_concatenate_with_existing_features(
self):
arff_data = arff.load("""@RELATION crosscheck
@ATTRIBUTE baseX NUMERIC
@ATTRIBUTE targetX NUMERIC
@ATTRIBUTE baseY NUMERIC
@ATTRIBUTE targetY NUMERIC
@ATTRIBUTE baseHeight NUMERIC
@ATTRIBUTE targetHeight NUMERIC
@ATTRIBUTE baseWidth NUMERIC
@ATTRIBUTE targetWidth NUMERIC
@ATTRIBUTE chiSquared NUMERIC
@ATTRIBUTE xbi {0,1}
@DATA
1,2,3,4,5,6,7,8,9,0
10,12,12,14,14,16,16,18,33,1""")
arff_data['data'] = np.array(arff_data['data'])
arff_data['X'] = np.array([[1, 2], [3, 4]])
arff_data['features'] = ['something', 'other']
extractor = CrossCheckExtractor(class_attr='xbi')
result = extractor.execute(arff_data)
self.assertEqual(2, len(result['X']))
self.assertEqual([
'something', 'other', 'area', 'displacement', 'sdr', 'chisquared'
], result['features'])
self.assertEqual(6, len(result['X'][0]))
self.assertEqual(1, result['X'][0][0])
self.assertEqual(2, result['X'][0][1])
self.assertEqual(35, result['X'][0][2])
self.assertAlmostEqual(math.sqrt(2), result['X'][0][3], places=2)
self.assertAlmostEqual((48 - 35) / 35, result['X'][0][4], places=2)
self.assertEqual(9, result['X'][0][5])
self.assertEqual(6, len(result['X'][1]))
self.assertEqual(3, result['X'][1][0])
self.assertEqual(4, result['X'][1][1])
self.assertEqual(224, result['X'][1][2])
self.assertAlmostEqual(2 * math.sqrt(2), result['X'][1][3], places=2)
self.assertAlmostEqual((16 * 18 - 14 * 16) / (14 * 16),
result['X'][1][4],
places=2)
self.assertEqual(33, result['X'][1][5])
self.assertEqual(2, len(result['y']))
self.assertEqual(0, result['y'][0])
self.assertEqual(1, result['y'][1])
self.assertEqual(arff_data['attributes'], result['attributes'])
def intializeBiases(cls, data, nrHidden):
# get the procentage of data points that have the i'th unit on
# and set the visible vias to log (p/(1-p))
percentages = data.mean(axis=0, dtype='float')
vectorized = np.vectorize(safeLogFraction, otypes=[np.float])
visibleBiases = vectorized(percentages)
hiddenBiases = np.zeros(nrHidden)
return np.array([visibleBiases, hiddenBiases])
def _read(self, g, f):
fi = self.ds.field_info[f]
if fi.particle_type and g.NumberOfParticles == 0:
# because this gets upcast to float
return np.array([],dtype='float64')
try:
temp = self.ds.index.io._read_data_set(g, f)
except:# self.ds.index.io._read_exception as exc:
if fi.not_in_all:
temp = np.zeros(g.ActiveDimensions, dtype='float64')
else:
raise
return temp
def rotate(self):
'method to rotate the given tetra in place using numpy'
print(self.getCoords())
#this makes sure the tetra cannot rotate itself out of bounds on the right edge
for i in self.getCoords():
if i[0] >9:
self.move('left')
#todo make sure the tetras rotated state will not overlap other tetras
self.activeTetra.orientation = np.rot90(np.array(self.activeTetra.orientation), 1)
self.render(self.activeTetra)
def __init__(self, patterns):
self.patterns = np.array(patterns)
def test_0D(self, sc, dtype):
self.assertIdenticalArray(sc(3), np.array(3, dtype=dtype))
def test_mixed_values(self, sc, dtype):
self.assertIdenticalArray(sc([1,2.3,4,5.6]), np.array([1,2.3,4,5.6], dtype=dtype))
def test_3D(self):
a3d = np.arange(12).reshape((2,3,2))
self.assertIdenticalArray(np(a3d), np.array(a3d))
def test_matrix_doublecolon(self, sc, dtype):
self.assertIdenticalArray(sc[1,2:
:3,4:
:5,6], np.array([[1,2],[3,4],[5,6]], dtype=dtype))
def main():
#### Data ####
# fake A/C params
pi=np.array([0.1,0.9])
a=np.array([[0.95,0.05],[0.05,0.95]])
mean=np.array([[0],[1500]])
cov=np.array([[[ 1.]],[[ 10]]])
model=hmm.GaussianHMM(pi.size, "full", pi,a)
model.means_ = mean
model.covars_ = cov
# randomly sample one day of data
length = 4 * 24
power, state = model.sample(length)
#### CNN ####
learning_rate = 0.1
rng = np.random.RandomState(23455)
ishape = (length, 1) # this is the size of our input data
batch_size = 20 # size of the minibatch
# allocate symbolic variables for the data
x = T.matrix('x') # generated power levels
y = T.lvector('y') # generate states
##############################
# BEGIN BUILDING ACTUAL MODEL
##############################
# Reshape matrix of rasterized images of shape (batch_size,length,1)
# to a 4D tensor, compatible with our ConvPoolLayer
layer0_input = x.reshape((batch_size,1,length,1))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (96-3+1,1-1+1)=(94,1)
# maxpooling reduces this further to (94/2,1/1) = (47,1)
# 4D output tensor is thus of shape (20,20,47,1)
layer0 = LeNetConvPoolLayer(rng, input=layer0_input,
image_shape=(batch_size, 1, length, 1),
filter_shape=(20, 1, 3, 1), poolsize=(3, 1))
# Construct the second convolutional pooling layer
# filtering reduces the image size to (12 - 5 + 1, 12 - 5 + 1)=(8, 8)
# maxpooling reduces this further to (8/2,8/2) = (4, 4)
# 4D output tensor is thus of shape (20,50,4,4)
#layer1 = LeNetConvPoolLayer(rng, input=layer0.output,
# image_shape=(batch_size, 20, 12, 12),
# filter_shape=(50, 20, 5, 5), poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size,num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (20, 20 * 47 * 1)
layer2_input = layer0.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = HiddenLayer(rng, input=layer2_input,
n_in=20*47*1, n_out=50,
activation=T.tanh )
# classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=50, n_out=1)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function([x, y], layer3.errors(y))
# create a list of all model parameters to be fit by gradient descent
params = layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by SGD
# Since this model has many parameters, it would be tedious to manually
# create an update rule for each model parameter. We thus create the updates
# dictionary by automatically looping over all (params[i],grads[i]) pairs.
updates = []
for param_i, grad_i in zip(params, grads):
updates.append((param_i, param_i - learning_rate * grad_i))
train_model = theano.function([index], cost, updates = updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]})
def test_mixed_values(self, sc, dtype):
self.assertIdenticalArray(sc[1,2.3:4,5.6], np.array([[1,2.3],[4,5.6]], dtype=dtype))
def test_3D(self):
a3d = np.arange(12).reshape((2,3,2))
self.assertIdenticalArray(np[[[ 0, 1], [ 2, 3], [ 4, 5]],
[[ 6, 7], [ 8, 9], [10, 11]]],
np.array(a3d))
def test_1D(self):
self.assertIdenticalArray(np([1,3,4,5,6,9]), np.array([1,3,4,5,6,9]))
def test_2D(self, sc, dtype):
a2d = np.arange(12).reshape((3,4))
self.assertIdenticalArray(sc(a2d), np.array(a2d, dtype=dtype))
def test_mixed_values(self):
self.assertIdenticalArray(np([1,2.3,4,5.6]), np.array([1,2.3,4,5.6]))
def test_3D(self, sc, dtype):
a3d = np.arange(12).reshape((2,3,2))
self.assertIdenticalArray(sc(a3d), np.array(a3d, dtype=dtype))
def test_float_values(self):
self.assertIdenticalArray(np([1.0, 2.0]), np.array([1.0,2.0]))
def test_1D(self, sc, dtype):
self.assertIdenticalArray(sc[1,3,4,5,6,9], np.array([1,3,4,5,6,9], dtype=dtype))
def test_float_values(self, sc, dtype):
self.assertIdenticalArray(sc[1.0, 2.0: 3.0, 4.0], np.array([[1.0, 2.0],[3.0,4.0]], dtype=dtype))
def test_mixed_values(self):
self.assertIdenticalArray(np.m[1,2.3:4,5.6], np.array([[1,2.3],[4,5.6]]))
from transform import four_point_transform
import np
import argparse
import cv2
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", help = "path to image file")
ap.add_argument("-c", "--coords", help = "comma seperated list of source pointer")
args = vars(ap.parse_args())
image = cv2.imread(args["image"])
pts = np.array(eval(args["coords"]), dtype = "float32")
warped = four_point_transform(image, pts)
cv2.imshow("original", image)
cv2.imshow("warped", warped)
cv2.waitKey(0)
def test_matrix_doublecolon(self):
self.assertIdenticalArray(np.m[1,2:
:3,4:
:5,6], np.array([[1,2],[3,4],[5,6]]))
def extractReal(img):
return np.array([[(img[j][i]).real for i in range(len(img))] for j in range(len(img[0]))])
def test_row(self):
self.assertIdenticalArray(np.m[1,2,3], np.array([[1,2,3]]))
def test_float_values(self):
self.assertIdenticalArray(np.m[1.0, 2.0: 3.0, 4.0], np.array([[1.0, 2.0],[3.0,4.0]]))
def test_float_values(self, sc, dtype):
self.assertIdenticalArray(sc([1.0, 2.0]), np.array([1.0,2.0], dtype=dtype))
def test_2D(self):
a2d = np.arange(12).reshape((3,4))
self.assertIdenticalArray(np(a2d), np.array(a2d))
def test_0D(self):
self.assertIdenticalArray(np(3), np.array(3))
def test_2D(self):
a2d = np.arange(12).reshape((3,4))
self.assertIdenticalArray(np[[0,1,2,3],[4,5,6,7],[8,9,10,11]],
np.array(a2d))