def main():
model = Model()
model.add(Conv2D(32, (3, 3), 'relu', 'he_normal', 'zeros', 1))
model.add(MaxPooling2D((2, 2), 2))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), 'relu', 'glorot_uniform', 'zeros', 1))
model.add(MaxPooling2D((2, 2), 2))
model.add(Dropout(0.25))
model.add(Conv2D(128, (3, 3), 'relu', 'glorot_uniform', 'zeros', 1))
model.add(Dropout(0.4))
model.add(Flattening())
model.add(Dense(128, 'relu', 'glorot_uniform', 'zeros'))
model.add(Dropout(0.3))
model.add(Dense(2, 'softmax', 'glorot_uniform', 'zeros'))
model.set_loss(CategoricalCrossEntropy)
data = get_data()
model.train(data,
32,
100,
'adam',
lr=1e-4,
beta1=0.9,
beta2=0.999,
epsilon=1e-7)
def create_dataLoader(opt):
(train_data, train_label), (validate_data, validate_label) = get_data(opt)
opt.train_count = len(train_data)
train_dataset = TDataSet(train_data, train_label, transform=toTensor)
train_loader = Data.DataLoader(dataset=train_dataset, batch_size=opt.batch_size, shuffle=True, drop_last=False)
opt.validate_count = len(validate_data)
validate_dataset = TDataSet(validate_data, validate_label, transform=toTensor)
validate_loader = Data.DataLoader(dataset=validate_dataset, batch_size=opt.batch_size, shuffle=False, drop_last=False)
return train_loader, validate_loader
import numpy as np
import math
import random
from sklearn.model_selection import GridSearchCV
import pickle
import statsmodels.api as sm
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
from sklearn.svm import SVR
from data_preprocess import get_data
# Data
(train_data, train_label), (test_data, test_label) = get_data()
from IPython import embed
# Random Forest
regr = RandomForestClassifier(n_jobs=-1)
num_estimator = [50,100,200,500]
max_feature = ["auto","sqrt","log2"]
min_samples_leaf = [10,50,100]
tuned_parameters = [{'n_estimators': num_estimator,'max_features':max_feature,'min_samples_leaf':min_samples_leaf}]
n_folds = 10
clf = GridSearchCV(regr, tuned_parameters, cv=n_folds)
# clf = RandomForestClassifier(n_jobs=-1, n_estimators=50, max_features='sqrt', min_samples_leaf=50)
new_columns.append(column)
return new_columns
def plot_statistic(statistic, title, names=None, conf_interv=None):
fig, ax = plt.subplots()
index = np.arange(len(statistic))
plt.bar(index, statistic, yerr=conf_interv, color='b', alpha=0.7)
if names is not None:
plt.xticks(index + 0.4, names)
# plt.xlabel("Stocks")
plt.ylabel(title)
plt.show()
data = data_preprocess.get_data(calc=True)
X = data.main_data_frame
def remove_outliers(data):
clf = OneClassSVM(nu=0.2, kernel="rbf", gamma=0.00001)
clf.fit(data)
logging.info("%s outliers removed from %s elements" % ((clf.predict(data) == -1).sum(), len(data)))
return data[clf.predict(data) == 1]
def clients_clusterization(X):
clients_df = X[get_columns(X, ['stock_id', 'customer_id', 'quantity', 'customer_status', 'license_type', 'iso',
'amount_in_usd', 'GDP', 'HDI', 'population', 'urban', 'avg_income',
'discount_perc'])]
def main():
print("starting test . . .")
if not checkIfNecessaryPathsAndFilesExist():
return
# end if
#get the class label strings
_, _, _, classifications = get_data()
# Build the graph for the cnn
x = tf.placeholder(tf.float32, [None, img_size, img_size, 3], name='in')
y_ = tf.placeholder(tf.int64, [None], name='y_')
y_conv, keep_prob = get_model(x, img_size, num_classes)
# 'Saver' op to save and restore all the variables
saver = tf.train.Saver()
# Softmax loss
with tf.name_scope('loss'):
cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
# accuracy
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_, name="final")
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
with tf.Session() as sess:
# Restore model weights from previously saved model
saver.restore(sess, model_path)
print("Model restored from file: %s" % model_path)
# for each file in the test images directory . . .
for fileName in os.listdir(TEST_IMAGES_DIR):
# if the file does not end in .jpg or .jpeg (case-insensitive), continue with the next iteration of the for loop
if not (fileName.lower().endswith(".jpg")
or fileName.lower().endswith(".jpeg")):
continue
# end if
# show the file name on std out
print(fileName)
# get the file name and full path of the current image file
imageFileWithPath = os.path.join(TEST_IMAGES_DIR, fileName)
# attempt to open the image with OpenCV
img = cv2.imread(imageFileWithPath)
openCVImage = cv2.resize(img, (img_size, img_size)).astype(
np.float32) / 255.0
# if we were not able to successfully open the image, continue with the next iteration of the for loop
if openCVImage is None:
print("unable to open " + fileName + " as an OpenCV image")
continue
# end if
# get the final tensor from the graph
finalTensor = sess.graph.get_tensor_by_name('y_:0')
# convert the OpenCV image (numpy array) to a TensorFlow image
tfImage = np.array(openCVImage)[:, :, 0:3]
tfImage = np.expand_dims(tfImage, axis=0)
# run the network to get the predictions
predictions = sess.run(tf.nn.softmax(y_conv), {
x: tfImage,
keep_prob: 1.0
})
# sort predictions from most confidence to least confidence
sortedPredictions = np.flip(predictions[0].argsort(),
0) #=[-len(predictions[0]):][::-1]
print("---------------------------------------")
# keep track of if we're going through the next for loop for the first time so we can show more info about
# the first prediction, which is the most likely prediction (they were sorted descending above)
onMostLikelyPrediction = True
# for each prediction . . .
for prediction in sortedPredictions:
strClassification = classifications[prediction]
# if the classification (obtained from the directory name) ends with the letter "s", remove the "s" to change from plural to singular
if strClassification.endswith("s"):
strClassification = strClassification[:-1]
# end if
# get confidence, then get confidence rounded to 2 places after the decimal
confidence = predictions[0][prediction]
# if we're on the first (most likely) prediction, state what the object appears to be and show a % confidence to two decimal places
if onMostLikelyPrediction:
# get the score as a %
scoreAsAPercent = confidence * 100.0
# show the result to std out
print("the object appears to be a " + strClassification +
", " + "{0:.2f}".format(scoreAsAPercent) +
"% confidence")
print(predictions)
# write the result on the image
writeResultOnImage(
img, strClassification + ", " +
"{0:.2f}".format(scoreAsAPercent) + "% confidence")
# finally we can show the OpenCV image
cv2.imshow(fileName, img)
# cv2.imwrite(RESULTS_DIR+ fileName,img)
# mark that we've show the most likely prediction at this point so the additional information in
# this if statement does not show again for this image
onMostLikelyPrediction = False
# end if
# for any prediction, show the confidence as a ratio to five decimal places
print(strClassification + " (" + "{0:.5f}".format(confidence) +
")")
# end for
# pause until a key is pressed so the user can see the current image (shown above) and the prediction info
cv2.waitKey()
# after a key is pressed, close the current window to prep for the next time around
cv2.destroyAllWindows()
# end for
# end with
# write the graph to file so we can view with TensorBoard
tfFileWriter = tf.summary.FileWriter(TENSORBOARD_DIR)
tfFileWriter.add_graph(sess.graph)
tfFileWriter.close()
import data_preprocess
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
from keras.utils import normalize
import numpy as np
import matplotlib.pyplot as plt
import cv2
X, y = data_preprocess.get_data()
model = Sequential()
model.add(Conv2D(64, (3, 3), input_shape=X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
#24x24x64
model.add(Conv2D(64, (3, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
# model.add(Dense(60))
# model.add( Activation("relu") )
model.add(Dense(1))
model.add(Activation("sigmoid"))