def __init__(self,
idx,
image,
model,
eta,
tau,
target_class,
bounds=(0, 1)):
self.IDX = idx
self.IMAGE = image
self.IMAGE_BOUNDS = bounds
self.MODEL = model
self.DIST_METRIC = eta[0]
self.DIST_VAL = eta[1]
self.TAU = tau
self.LABEL, _ = self.MODEL.predict(self.IMAGE)
feature_extraction = FeatureExtraction(pattern='grey-box')
self.PARTITIONS = feature_extraction.get_partitions(self.IMAGE,
self.MODEL,
num_partition=10)
self.DIST_EVALUATION = {}
self.ADV_MANIPULATION = ()
self.ADVERSARY_FOUND = None
self.ADVERSARY = None
self.current_d = [0]
self.target_class = target_class
print("Distance metric %s, with bound value %s." %
(self.DIST_METRIC, self.DIST_VAL))
def main():
# perform feature extraction to get a file containing
# the features of all models
FeatureExtraction.featureExtraction()
# load the features
X, Y = load_svmlight_file("features/features.txt")
# create the classifier and set the gamma value
clf = svm.SVC(gamma=0.0001, C=100)
# set the cross validation to be leave-one-out
cv = LeaveOneOut(len(Y))
# exhaustive grid search
# tuned_parameters = [{'kernel': ['rbf'], 'gamma': [.005, .0005],
# 'C': [1, 10, 100, 1000]}]
# clf = GridSearchCV(SVC(C=1), tuned_parameters, cv=cv, scoring='accuracy')
# clf.fit(X, Y)
# print(clf.best_estimator_)
# for params, mean_score, scores in clf.grid_scores_:
# print("%0.3f (+/-%0.03f) for %r"
# % (mean_score, scores.std() / 2, params))
# calculate the accuracy
scores = cval.cross_val_score(clf, X, Y, cv=cv, scoring='accuracy')
print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
def Execute(self):
images = self.GetInputStageValue(0, "images")
self.StartProcess()
kds = []
images.WriteFileList()
ds = 0
if (self._properties["Downsample"]): ds = 1
for im in images.GetImages():
keypointDescriptorFile = os.path.join(
os.path.splitext(im.GetFilePath())[0] + ".key")
if (Common.Utility.ShouldRun(self._properties["Force Run"],
keypointDescriptorFile)):
self.RunCommand(
"ASIFTkeypoint",
Common.Utility.CommandArgs(
Common.Utility.Quoted(im.GetFilePath()),
Common.Utility.Quoted(keypointDescriptorFile),
self._properties["Number of Tilts"], ds))
kd = FeatureExtraction.KeypointDescriptorFileLowe(
keypointDescriptorFile, False)
kds.append(kd)
kds = FeatureExtraction.KeypointDescriptors(images.GetPath(), kds)
self.SetOutputValue("keypointDescriptors", kds)
def submit1():
train = pd.read_csv(train_path, header=0, index_col=0)
test = pd.read_csv(test_path, header=0, index_col=0)
train_test = {'train': train, 'test': test}
for key in train_test:
data = train_test[key]
logging.info(
'------getting FamilySize feature for {}-----'.format(key))
data['Family_Size'] = FE.getFamilySize(data)
logging.info(
'-------done getting FamilySize feature for {} -----'.format(key))
logging.info('------getting IsAlone feature for {}-----'.format(key))
data['IsAlone'] = FE.getIsAlone(data)
logging.info(
'------done getting IsAlone feature for {}------'.format(key))
y = train.pop("Survived").values
X = train.values
rf = RandomForestClassifier(100)
bg = GradientBoostingClassifier(n_estimators=100)
xg = xgb.XGBClassifier(n_estimators=100)
tree = DecisionTreeClassifier()
stacking = MF.StackingModel([rf, bg, xg], tree, cv=5)
stacking.fit(X, y)
my_prediction = stacking.predict(test.values)
passengerId = np.arange(892, 892 + 418)
prediction_path = os.path.join(os.path.dirname(os.getcwd()),
'data/gender_submission.csv ')
pd.DataFrame({
'PassengerId': passengerId,
'Survived': my_prediction
}).to_csv(prediction_path, index=False)
logging.info('Successfully done')
def Execute(self):
images = self.GetInputStageValue(0, "images")
self.StartProcess()
kds = []
for im in images.GetImages():
keypointDescriptorFile = os.path.join(
os.path.splitext(im.GetFilePath())[0] + ".key")
if (Common.Utility.ShouldRun(self._properties["Force Run"],
keypointDescriptorFile)):
daisyKD = self.Process(im.GetFilePath())
kd = FeatureExtraction.KeypointDescriptorFileLowe(
daisyKD, self._properties["Parse Descriptors"])
kd.Write(os.path.splitext(daisyKD.GetFilePath())[0] + ".key")
else:
kd = FeatureExtraction.KeypointDescriptorFileLowe(
keypointDescriptorFile,
self._properties["Parse Descriptors"])
kds.append(kd)
kds = FeatureExtraction.KeypointDescriptors(images.GetPath(), kds)
self.SetOutputValue("keypointDescriptors", kds)
def __init__(self, image, model, eta, tau, bounds=(0, 1)):
self.IMAGE = image
self.IMAGE_BOUNDS = bounds
self.MODEL = model
self.DIST_METRIC = eta[0]
self.DIST_VAL = eta[1]
self.TAU = tau
self.LABEL, _ = self.MODEL.predict(self.IMAGE)
feature_extraction = FeatureExtraction(pattern='grey-box')
self.PARTITIONS = feature_extraction.get_partitions(self.IMAGE,
self.MODEL,
num_partition=10)
self.ALPHA = {}
self.BETA = {}
self.MANI_BETA = {}
self.MANI_DIST = {}
self.CURRENT_MANI = ()
self.ROBUST_FEATURE_FOUND = False
self.ROBUST_FEATURE = []
self.FRAGILE_FEATURE_FOUND = False
self.LEAST_FRAGILE_FEATURE = []
print("Distance metric %s, with bound value %s." %
(self.DIST_METRIC, self.DIST_VAL))
def detectEmblemSIFT(imagePath, precision, updateTrainingData):
image = cv2.imread(imagePath)
if updateTrainingData:
updateTrainingData()
templates = fe.createTemplateListFeatureExtraction(False)
templatesCopy = copy.deepcopy(templates)
templatesCopy = sorted(templatesCopy,
key=lambda template: template.positions,
reverse=True)
fe.detectKeyPointsWithSIFT(image, templatesCopy, True, precision)
for template in templatesCopy:
imageCopy = image.copy()
template.calculateConfidence(templatesCopy, 0.00)
#for position in template.positions:
# print a rectangle for each found position
# cv2.rectangle(imageCopy, (position[0]-5,position[1]-5), (position[0]+5,position[1]+5), (0, 0, 255), 2)
# Show image
#cv2.imshow(template.name,imageCopy)
#cv2.waitKey()
templatesCopy = sorted(templatesCopy,
key=lambda template: len(template.positions),
reverse=True)
for template in templatesCopy:
print template.name, 'emblem is found with confidence of:', len(
template.positions)
def compute_chunk_features(mp3_file):
"""Return feature vectors for two chunks of an MP3 file."""
# Extract MP3 file to a mono, 10kHz WAV file
mpg123_command = ('D:\Softwares\Audio_Video_Tools\mpg123-1.22.0-x86-64'
'\mpg123.exe -w "%s" -r 10000 -m "%s"')
out_file = 'temp.wav'
cmd = mpg123_command % (out_file, mp3_file)
temp = subprocess.call(cmd)
# Read in chunks of data from WAV file
wav_data1, wav_data2 = read_wav(out_file)
# We'll cover how the features are computed in the next section!
return FeatureExtraction.features(wav_data1), FeatureExtraction.features(wav_data2)
def object_matching(img1, img2, method='sift', max_points=50):
MIN_MATCH_COUNT = 10
kp1, des1 = FeatureExtraction.feature_extraction(img1, method=method)
kp2, des2 = FeatureExtraction.feature_extraction(img2, method=method)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)
matches = sorted(matches, key=lambda x: x[1].distance)
# store all the good matches as per Lowe's ratio test.
good = []
for m, n in matches:
if m.distance < 0.7 * n.distance:
good.append(m)
good = good[:min(max_points, len(good))]
if len(good) > MIN_MATCH_COUNT:
src_pts = np.float32([kp1[m.queryIdx].pt
for m in good]).reshape(-1, 1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt
for m in good]).reshape(-1, 1, 2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
matchesMask = mask.ravel().tolist()
h, w = img1.shape[:2]
pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1],
[w - 1, 0]]).reshape(-1, 1, 2)
dst = cv2.perspectiveTransform(pts, M)
img2 = cv2.polylines(img2, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)
else:
print("Not enough matches are found - %d/%d" %
(len(good), MIN_MATCH_COUNT))
matchesMask = None
draw_params = dict(
matchColor=(0, 255, 0), # draw matches in green color
singlePointColor=None,
matchesMask=matchesMask, # draw only inliers
flags=2)
img3 = cv2.drawMatches(img1, kp1, img2, kp2, good, None, **draw_params)
cv2.namedWindow('image', cv2.WINDOW_AUTOSIZE)
cv2.imshow('image', img3)
cv2.waitKey()
cv2.destroyAllWindows()
return img3
def modelTest3():
train_path = os.path.join(os.path.dirname(cwd), 'data/train_.csv')
train = pd.read_csv(train_path, header=0, index_col=0)
train['FamilySize'] = FE.getFamilySize(train)
train['IsAlone'] = FE.getIsAlone(train)
y = train.pop("Survived").values
X = train.values
rf = RandomForestClassifier(100)
bg = GradientBoostingClassifier(n_estimators=100)
xg = xgb.XGBClassifier(n_estimators=100)
tree = DecisionTreeClassifier()
stacking = StackingModel([rf, bg, xg], tree, cv=5)
scores = cross_val_score(stacking, X=X, y=y, cv=5)
print('modelTest2:', np.mean(scores))
def image_process(image_list):
features = []
for image in image_list:
normalized_image = IrisNormalization.iris_normalization(image)
enhanced = ImageEnhancement.enhancement(normalized_image, 32)
sigma_x1, sigma_y1 = 3.0, 1.5
sigma_x2, sigma_y2 = 4.5, 1.5
roi_1 = FeatureExtraction.spatial_filter(enhanced, sigma_x1, sigma_y1,
4, 4)[0:48, :]
roi_2 = FeatureExtraction.spatial_filter(enhanced, sigma_x2, sigma_y2,
4, 4)[0:48, :]
features.append(
FeatureExtraction.feature_extraction(8, roi_1) +
FeatureExtraction.feature_extraction(8, roi_2))
return np.array(features)
def testImage(image_list):
global arr1
global flag
arr1.clear()
for index in range(len(image_list)):
img = io.imread(image_list[index], as_grey=True)
infile = cv2.imread(image_list[index])
infile = infile[:, :, 0]
hues = (np.array(infile) / 255.) * 179
outimageHSV = np.array([[[b, 255, 255] for b in a]
for a in hues]).astype(int)
outimageHSV = np.uint8(outimageHSV)
outimageBGR = cv2.cvtColor(outimageHSV, cv2.COLOR_HSV2BGR)
rgb = io.imread(image_list[index])
lab = color.rgb2lab(rgb)
outimageBGR = lab
for i in range(outimageBGR.shape[0]):
for j in range(outimageBGR.shape[1]):
sum = 0
for k in range(outimageBGR.shape[2]):
sum = sum + outimageBGR[i][j][k]
sum = sum / (3 * 255)
if (i < img.shape[0] and j < img.shape[1]):
img[i][j] = sum
S = preprocessing.MinMaxScaler((0, 19)).fit_transform(img).astype(int)
Grauwertmatrix = feature.greycomatrix(
S, [1, 2, 3], [0, np.pi / 4, np.pi / 2, 3 * np.pi / 4],
levels=20,
symmetric=False,
normed=True)
arr1.append(feature.greycoprops(Grauwertmatrix, 'contrast'))
arr1.append(feature.greycoprops(Grauwertmatrix, 'correlation'))
arr1.append(feature.greycoprops(Grauwertmatrix, 'homogeneity'))
arr1.append(feature.greycoprops(Grauwertmatrix, 'ASM'))
arr1.append(feature.greycoprops(Grauwertmatrix, 'energy'))
arr1.append(feature.greycoprops(Grauwertmatrix, 'dissimilarity'))
arr1.append(Features.sumOfSquares(Grauwertmatrix))
arr1.append(Features.sumAverage(Grauwertmatrix))
arr1.append(Features.sumVariance(Grauwertmatrix))
arr1.append(Features.Entropy(Grauwertmatrix))
arr1.append(Features.sumEntropy(Grauwertmatrix))
arr1.append(Features.differenceVariance(Grauwertmatrix))
arr1.append(Features.differenceEntropy(Grauwertmatrix))
arr1.append(Features.informationMeasureOfCorelation1(Grauwertmatrix))
arr1.append(Features.informationMeasureOfCorelation2(Grauwertmatrix))
flag = 1
def loc_norm_extract(folder_path):
image_total_path = readFromPath(folder_path)
print(len(image_total_path))
ii = 0
num = 0
X = []
y = []
for image_path in image_total_path:
clas = int(image_path[-11:-8])
img_orig = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
img_cut = loc.roughLocalization(img_orig)
laplacian, center = loc.binarizeEdgeDetection(img_cut)
row_x, col_y, r, pupil = loc.innerBoundComputation(
img_cut, laplacian, center)
img_cut2, row_x1, col_y1 = loc.shrink(img_cut, row_x, col_y, r)
[row_x_o, col_y_o,
r_o], dist_o, img_outer = loc.cannyEdge(img_cut2, row_x1, col_y1, r)
img_norm_list = norm.rotateNormalization(img_cut2, [col_y1, row_x1, r],
[col_y_o, row_x_o, r_o])
# cv2.imwrite("train_pupil"+ image_path[image_path.rfind("/"):image_path.rfind(".") ] +"_p.bmp" ,pupil)
# cv2.imwrite("train_outer"+ image_path[image_path.rfind("/"):image_path.rfind(".") ] +"_o.bmp" ,img_outer)
for i, img_norm in enumerate(img_norm_list):
img_enhance = enhance.ImageEnhancement(img_norm)
FeatureList = extract.featureExtraction(img_enhance)
X.append(FeatureList)
y.append(clas)
print(ii)
ii += 1
return X, y
def select_image():
img_path = filedialog.askopenfilename()
image = Image.open(img_path)
image = image.resize((250, 250), Image.ANTIALIAS)
image = ImageTk.PhotoImage(image)
panel = Label(root, image=image)
panel.image = image
panel.grid(column=2)
# l_features, s_features = FeatureExtraction.separate_leaf_and_sheath(cv2.imread(img_path), predict=True)
# print(l_features)
# # new_f = []
# # for l in l_features:
# # temp = []
# # temp.append(l[1])
# # temp.append(l[3])
# # temp.append(l[4])
# # temp.append(l[6])
# # new_f.append(temp)
l_features, s_features = FeatureExtraction.predict_normal_features(
cv2.imread(img_path))
result = SVMModel.predict_normal_leaf_model(l_features)
print(result)
def feature_extraction_for_each(wav_files, feature="mfcc"):
for file in wav_files:
(sample_rate, speech_signal) = wavreader.read("{}wav/{}/{}.wav".format(
DIR_PATH, file[0], file[1]))
feature_vector = feat.mfcc(speech_signal, sample_rate, 0.016, 0.008)
def predict(url):
# print("HI")
feature = FeatureExtraction.extractFeatures(url)
#print(feature)
prediction1 = classifier1.predict([feature[1:]])
prediction2 = classifier1.predict([feature[1:]])
return (prediction1[0], prediction2[0])
def start(self):
# perform some logging
self.jlogger.info("Starting job with job id {}".format(self.job_id))
self.jlogger.debug("Job Config: {}".format(self.config))
self.jlogger.debug("Job Other Data: {}".format(self.job_data))
try:
rud.ReadUserData(self)
fg.FeatureGeneration(self, is_train=True)
pp.Preprocessing(self, is_train=True)
fs.FeatureSelection(self, is_train=True)
fe.FeatureExtraction(self, is_train=True)
clf.Classification(self)
cv.CrossValidation(self)
tsg.TestSetGeneration(self)
tspp.TestSetPreprocessing(self)
tsprd.TestSetPrediction(self)
job_success_status = True
except:
job_success_status = False
helper.update_running_job_status(self.job_id, "Errored")
self.jlogger.exception("Exception occurred in ML Job {} ".format(
self.job_id))
return job_success_status
def Execute(self):
images = self.GetInputStageValue(0, "images")
self.StartProcess()
q = multiprocessing.Queue()
for i,im in enumerate(images.GetImages()):
q.put_nowait((i,im))
kds = [None]*len(images.GetImages())
workers = []
for i in range(self._properties["CPUs"]):
w = Sift.Worker(q, kds, self)
w.start()
workers.append(w)
errors = []
for w in workers:
w.join()
errors.extend(w.getErrors())
if (len(errors)>0):
raise Exception(string.join(errors,"\n"))
kds = FeatureExtraction.KeypointDescriptors(images.GetPath(), kds)
self.SetOutputValue("keypointDescriptors", kds)
def getResult(url):
#Importing dataset
data = np.loadtxt("dataset.csv", delimiter=",")
#Seperating features and labels
X = data[:, :-1]
y = data[:, -1]
#Seperating training features, testing features, training labels & testing labels
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
clf = rfc()
clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)
print(score * 100)
X_new = []
X_input = url
X_new = FeatureExtraction.generate_data_set(X_input)
X_new = np.array(X_new).reshape(1, -1)
try:
prediction = clf.predict(X_new)
if prediction == -1:
return "Phishing Url"
else:
return "Legitimate Url"
except:
return "Phishing Url"
def predict(url):
# print("HI")
feature = FeatureExtraction.extractFeatures(url)
print(feature)
prediction = classifier.predict([feature[1:]])
print(prediction[0])
return (prediction[0])
def process(self, im):
# special case
if (self.__parent._properties["Sift Method"] == "VLFeat"):
exeName = "sift"
argsPattern = "--orientations \"%s\" -o \"%s\""
keypointDescriptorFile = os.path.join(os.path.splitext(im.GetFilePath())[0]+".key")
if (Common.Utility.ShouldRun(self.__parent._properties["Force Run"], keypointDescriptorFile)):
self.__parent.RunCommand(exeName,
argsPattern % (im.GetFilePath(),keypointDescriptorFile))
vlkd = KeypointDescriptorFileVLFeat(keypointDescriptorFile, True)
kd = FeatureExtraction.KeypointDescriptorFileLowe(vlkd)
kd.Write(vlkd.GetFilePath())
else:
kd = FeatureExtraction.KeypointDescriptorFileLowe(keypointDescriptorFile,
self.__parent._properties["Parse Descriptors"])
return kd
else:
if (self.__parent._properties["Sift Method"] == "SiftWin32"):
exeName = "siftWin32"
argsPattern = "<\"%s\"> \"%s\""
elif (self.__parent._properties["Sift Method"] == "SiftHess"):
exeName = "sifthess"
argsPattern = "\"%s\" \"%s\""
elif (self.__parent._properties["Sift Method"] == "SiftGPU"):
exeName = "SiftGPUKeypoint"
argsPattern = "\"%s\" \"%s\""
else:
raise Exception("Unknown Sift method: " + self.__parent._properties["Sift Method"])
keypointDescriptorFile = os.path.join(os.path.splitext(im.GetFilePath())[0]+".key")
if (Common.Utility.ShouldRun(self.__parent._properties["Force Run"], keypointDescriptorFile)):
self.__parent.RunCommand(exeName,
argsPattern % (im.GetFilePath(),keypointDescriptorFile))
kd = FeatureExtraction.KeypointDescriptorFileLowe(keypointDescriptorFile,
self.__parent._properties["Parse Descriptors"])
return kd
def write_to_csv_articles(titlecsv, publisher):
# if publisher:
# xml_gt = etree.iterparse(path_publisher_gt, tag='article')
# xml_training = etree.iterparse(path_publisher_training, tag='article')
# content_training = etree.iterparse(path_publisher_training, tag='article')
# else:
# xml_gt = etree.iterparse(path_article_gt, tag='article')
# xml_training = etree.iterparse(path_article_training, tag='article')
# content_training = etree.iterparse(path_article_training, tag='article')
#------------------Uncomment this to parse the Validation-Datasets------------------------------------------------------
xml_gt = etree.iterparse(path_validation_gt, tag='article')
xml_training = etree.iterparse(path_validation_training, tag='article')
content_training = etree.iterparse(path_validation_training, tag='article')
#-----------------------------------------------------------------------------------------------------------------------
feature_extraction.parse_features(xml_training, publisher)
groundtruth_parser.parse_groundtruth(xml_gt, publisher)
content = content_parser.parse_content(content_training)
id = feature_extraction.get_id_array(publisher)
published = feature_extraction.get_published_at_array(publisher)
title = feature_extraction.get_title_array(publisher)
bias = groundtruth_parser.get_bias_array(publisher)
hyperpartisan = groundtruth_parser.get_hyperpartisan_array(publisher)
#------------------Uncomment this to for the Validation-Datasets--------------------------------------------------------
published = []
for _ in range(150000-len(published)):
published.append('/')
#-----------------------------------------------------------------------------------------------------------------------
columns = {"ArticleID": id,
"PublishedAt": published,
"Title": title,
"Bias": bias,
"Content": content,
"Hyperpartisan": hyperpartisan}
table_frame = Pd.DataFrame(columns)
table_frame = table_frame[['ArticleID', 'PublishedAt', 'Title', 'Bias', 'Content', 'Hyperpartisan']]
print(table_frame)
table_frame.to_csv(titlecsv, encoding='utf-8', index=False)
def match(self, image, topn=5):
features = moduleFE.extract_features(image)
img_distances = self.cos_cdist(features)
# getting top 5 records
nearest_ids = np.argsort(img_distances)[:topn].tolist()
nearest_img_paths = self.names[nearest_ids].tolist()
return nearest_img_paths, img_distances[nearest_ids].tolist()
def process_df_frames_des(item, method='sift'):
import FeatureExtraction
filename, df_frames = item
if df_frames['frame_index'].iloc[0] % 50 != 0:
return [], []
img = Preprocessing.load_image(filename)
kps, dess = FeatureExtraction.feature_extraction(img, method)
return kps, dess
def keymatchChains(imagePath):
# KeyMatchFull, KeyMatchGPU
imageSource = Sources.ImageSource(imagePath, "pgm")
sift = FeatureExtraction.Sift(imageSource, False, "SiftHess")
keyMatch = FeatureMatch.KeyMatch(sift, True, "KeyMatchFull", forceRun=True)
print Chain.Render(keyMatch,"UnitTest-keymatchChains-KeyMatchFull-log.txt")
def testing(path):
feature = fe.getCSVFeatures(path)
if not (os.path.exists('DataSet/TestFeatures')):
os.makedirs('DataSet/TestFeatures')
with open('DataSet\\TestFeatures/testcsv.csv', 'w') as handle:
handle.write(
'ratio,cent_y,cent_x,eccentricity,solidity,skew_x,skew_y,kurt_x,kurt_y\n'
)
handle.write(','.join(map(str, feature)) + '\n')
def predict(url):
# print("HI")
feature = FeatureExtraction.main(url)
print(url, feature)
prediction1 = classifier1.predict(feature)
prediction2 = classifier2.predict(feature)
prediction3 = classifier3.predict(feature)
return (prediction1[0],prediction2[0],prediction3[0])
def main(model_path, input_data_path, labels_data_path, vinput_data_path,
vlabels_data_path, saved_features_path):
# Create directory to save the learned model in given model_path
if not os.path.exists(model_path):
try:
os.makedirs(model_path)
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
# Load and format data to prepare for feature extraction
tweets = pickle.load(open('../data/xtrain.p', 'rb'))
labels = pickle.load(open('../data/ytrain.p', 'rb'))
vinputs = pickle.load(open('../data/xvalid.p', 'rb'))
vlabels = pickle.load(open('../data/yvalid.p', 'rb'))
print("Extracting Features ...")
M, word_vectorizer, char_vectorizer, full_vectorizer, names = FeatureExtraction(
tweets)
Mp = FeatureExtractionTest(vinputs, word_vectorizer, char_vectorizer,
full_vectorizer)
# Train model
print("Training Model ...")
model1 = LogisticRegression(penalty='l2',
class_weight='balanced',
C=0.95,
solver='liblinear',
multi_class='ovr')
model2 = RandomForestClassifier(n_estimators=100,
class_weight={
0: 0.85,
1: 0.05,
2: 0.10
})
model = VotingClassifier(estimators=[('lr', model1), ('rf', model2)],
voting='hard',
weights=[1, 1]).fit(M, labels)
l_predict = model.predict(M)
f1score = f1_score(labels, l_predict, average=None)
print(f1score)
print(classification_report(labels, l_predict))
predicted_labels = model.predict(Mp)
# Compute FScore
f1score = f1_score(vlabels, predicted_labels, average=None)
print(f1score)
print(classification_report(vlabels, predicted_labels))
# Save trained model
trained_model_filename = model_path + "model.sav"
joblib.dump(model, trained_model_filename)
def identifyWriterSVM(svclassifier,
methods,
writerImage,
writerImageGray,
classifiedCO3=[],
classifiedSift=[]):
featureVector = featureExtraction.extractAndConcatinateFeauturesDuringIdentification(
methods, writerImage, writerImageGray, classifiedCO3, classifiedSift)
nearestWriter = svclassifier.predict([featureVector])
return nearestWriter
def pr():
print('\n')
print("Contrast : %f" % (np.mean(ContrastStats)))
print("Energy : %f" % (np.mean(Energy)))
print("Dissimilarity : %f" % (np.mean(Dissimilarity)))
print("Correlation : %f" % (np.mean(CorrelationtStats)))
print("Homogeneity : %f" % (np.mean(ASMStats)))
print("Sum of Squares : %f" % (np.mean(Features.sumOfSquares(Grauwertmatrix))))
print("Sum Average : %f" % (np.mean(Features.sumAverage(Grauwertmatrix))))
print("Sum Variance : %f" % (np.mean(Features.sumVariance(Grauwertmatrix))))
print("Sum Entropy : %f" % (np.mean(Features.sumEntropy(Grauwertmatrix))))
print("Entropy : %f" % (np.mean(Features.Entropy(Grauwertmatrix))))
print("Difference Variance : %f" % (np.mean(Features.differenceVariance(Grauwertmatrix))))
print("Difference Entropy : %f" % (np.mean(Features.differenceEntropy(Grauwertmatrix))))
print("Information measure of corelation1: %f" % (np.mean(Features.informationMeasureOfCorelation1(Grauwertmatrix))))
print("Information measure of corelation2: %f" % (np.mean(Features.informationMeasureOfCorelation2(Grauwertmatrix))))
def classifiy(class_num, subsample_size, window_size, cluster_num, max_iter, rnd_number, neighbor_num, train_X, train_y, test_X, test_y):
print("=== Classify Start ===\n")
features = FeatureExtraction.read_features(FeatureExtraction.gen_feature_fname(class_num, subsample_size, window_size, cluster_num))
knn = KNeighborsClassifier(n_neighbors=neighbor_num)
feature_vectors_train = FeatureExtraction.trainset_project(features, train_X, True)
feature_vectors_test = FeatureExtraction.trainset_project(features, test_X, True)
start = time.time()
knn.fit(feature_vectors_train, train_y)
print("KNN Fit Time: ",time.time()-start)
start = time.time()
acc = 0
test_len = len(feature_vectors_test)
for i in range(0, test_len):
if knn.predict(feature_vectors_test[i]) == test_y[i]:
acc += 1
# end if
# end for
print("KNN Predict Time: ",time.time()-start)
print("Accuracy: ",(acc/test_len),", Neighbor Number: ",neighbor_num)
print("=== Classify Finish ===")
def createFeatureDict(dataBin):
temp = []
featureExtDict = {}
for i in range(0,len(movements)):
temp = []
tempEMGData = convertToIntMatrix(dataBin[movements[i]].getEMGData())
for num in featuresToExtract:
temp.append(featExt.selectFeatures(None,tempEMGData,number = num))
featureExtDict[movements[i]] = temp
return featureExtDict
def siftChains(imagePath):
#SiftWin32, SiftHess, SiftGPU, VLFeat
imageSource = Sources.ImageSource(imagePath, "pgm")
sift = FeatureExtraction.Sift(imageSource, False, "SiftWin32", forceRun=True)
# SiftWin32 only on windows
if (Common.Utility.OSName=="Windows"):
print Chain.Render(sift,"UnitTest-siftChains-SiftWin32-log.txt")
sift.SetProperty("Sift Method", "VLFeat")
print Chain.Render(sift,"UnitTest-siftChains-VLFeat-log.txt")
# daisy only on windows (note: this should not be the last test, as the descriptors are for ROIs)
if (Common.Utility.OSName=="Windows"):
imageSource = Sources.ImageSource(imagePath, "jpg")
daisy = FeatureExtraction.Daisy(imageSource, False, roi="0,0,50,50", forceRun=True)
print Chain.Render(daisy,"UnitTest-siftChains-daisy-log.txt")
sift.SetProperty("Sift Method", "SiftHess")
print Chain.Render(sift,"UnitTest-siftChains-SiftHess-log.txt")
autoload = False # When i dont want to waste time on repeated computation of attribute values
df_all = None
num_train = 0
def check_preprocessed_csv_exists(name):
import os
for file in os.listdir(os.getcwd()):
if file.find(name) >= 0:
return True
return False
if not autoload: # compute attribute values manually
df_all = FeatureExtraction.extract_features()
num_train = FeatureExtraction.df_train.shape[0]
df_all.to_csv('my_df_all_naheed.csv')
else:
if check_preprocessed_csv_exists("my_df_all_naheed.csv"):
df_all = FeatureExtraction.autoload_featurevectors(
"my_df_all_naheed.csv") # autoload feature vectors from csv file
num_train = FeatureExtraction.df_train.shape[0]
assert df_all is not None
print df_all.axes
df_train = df_all.iloc[:num_train]
df_test = df_all.iloc[num_train:]
def grid_search_for_cluster(class_num, subsample_size, window_size, max_iter, rnd_number, cluster_num_seq):
for cluster_num in cluster_num_seq:
FeatureExtraction.extract_feature_by_kmeans(
class_num, subsample_size, window_size, cluster_num, max_iter, rnd_number
)