def train(self):
logdir = self.model_dir + datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S")
hr_images_test, lr_images_test, hr_images_train, lr_images_train = images_loader_mini(
self.input_dir, self.scale)
y_train_hr = np.array(hr_images_train[:500000])
x_train_lr = np.array(lr_images_train[:500000])
y_test_hr = np.array(hr_images_test[:15000])
x_test_lr = np.array(lr_images_test[:15000])
y_train_hr = normalize(y_train_hr)
y_test_hr = normalize(y_test_hr)
x_train_lr = normalize(x_train_lr)
x_test_lr = normalize(x_test_lr)
model = SRDeepCNN(self.channels, self.scale).build_model()
model.compile(loss=content_loss,
optimizer=get_optimizer(),
metrics=[metrics.mse, metrics.categorical_accuracy])
tensorboard_callback = keras.callbacks.TensorBoard(
log_dir=logdir,
batch_size=self.batch_size,
write_graph=True,
write_images=True,
write_grads=True)
loss_history = model.fit(x_train_lr,
y_train_hr,
batch_size=self.batch_size,
epochs=self.epochs,
verbose=1,
validation_data=([x_test_lr, y_test_hr]),
callbacks=[tensorboard_callback])
save_model(model, loss_history, self.model_dir)
plot_generated_test(self.output_dir, model, y_test_hr, x_test_lr)
def train():
algorithm = "GMM"
n_cluster = 7
xy = np.loadtxt('faults.csv', delimiter=',') #1941
x_data = xy[:, :-n_cluster]
if use_normalize:
normalize(x_data)
pandas_data = pd.DataFrame(data=x_data)
if algorithm == "K-Means":
result = KMeans(n_clusters=n_cluster,
init='k-means++',
max_iter=300,
random_state=random_Seed).fit(pandas_data).labels_
elif algorithm == "GMM":
gmm = GaussianMixture(n_components=7, random_state=random_Seed)
gmm_label = gmm.fit(pandas_data).predict(pandas_data)
result = gmm_label
else:
return
pandas_data['cluster'] = result
pca = PCA(n_components=2)
pca_transformed = pca.fit_transform(pandas_data)
pandas_data['pca_x'] = pca_transformed[:, 0]
pandas_data['pca_y'] = pca_transformed[:, 1]
pandas_data.head(3)
marker_ind = []
for i in range(n_cluster):
marker_ind.append(pandas_data[pandas_data['cluster'] == i].index)
for i in range(n_cluster):
plt.scatter(x=pandas_data.loc[marker_ind[i], 'pca_x'],
y=pandas_data.loc[marker_ind[i], 'pca_y'],
marker=marker[i])
plt.xlabel('PCA 1')
plt.ylabel('PCA 2')
plt.title(
'7 Clusters Visualization by 2 PCA Components({0})'.format(algorithm))
plt.show()
def predict(self, image_matrix):
X = image_matrix.reshape((1, image_matrix.size))
mu = np.loadtxt('mu.csv')
sigma = np.loadtxt('sigma.csv')
X, mu, sigma = normalize(X, mu, sigma)
hidden_size = self.config['hidden_size']
input_size = self.config['input_size']
num_labels = self.config['num_labels']
theta1 = self.nn_params[:((hidden_size) * (input_size + 1))].reshape(
(hidden_size, input_size + 1))
theta2 = self.nn_params[((hidden_size) * (input_size + 1)):].reshape(
(num_labels, hidden_size + 1))
a1 = insert_bias(X)
z2 = theta1.dot(a1.T)
a2 = sigmoid(z2)
a2 = insert_bias(a2.T)
print theta2.shape
z3 = theta2.dot(a2.T)
h = sigmoid(z3)
print h
return h
def _solve(self, rule, query):
ops = []
facts = []
for f in rule:
if f.isalpha():
facts.append(f)
else:
ops.append(f)
if ops[0] == self.config.op["and"]:
res = op_and(self.data[facts[0]], self.data[facts[0]])
res = normalize(res)
for i, op in enumerate(ops[1:]):
if ops[0] == self.config.op["and"]:
res = op_and(res, self.data[facts[i + 2]])
res = normalize(res)
self.data[query] = res
def main():
X_person, y_person = [], []
X_org, y_org = [], []
X_loc, y_loc = [], []
for input in input_list:
entities, person_mat, org_mat, loc_mat = compute_features(input)
normalize(person_mat)
normalize(org_mat)
normalize(loc_mat)
Xys = generate_diff_data(entities, person_mat, org_mat, loc_mat)
X_person += Xys[0][0]
y_person += Xys[0][1]
X_org += Xys[1][0]
y_org += Xys[1][1]
X_loc += Xys[2][0]
y_loc += Xys[2][1]
print "Training Person Ranks..."
clf_person = svm.SVC(C=1, kernel='linear')
clf_person.fit(X_person, y_person)
print "Training Error = %.2f%%" % (training_error(X_person, y_person, clf_person) * 100)
print "Training Organization Ranks..."
clf_org = svm.SVC(C=1, kernel='linear')
clf_org.fit(X_org, y_org)
print "Training Error = %.2f%%" % (training_error(X_org, y_org, clf_org) * 100)
print "Training Location Ranks..."
clf_loc = svm.SVC(C=1, kernel='linear')
clf_loc.fit(X_loc, y_loc)
print "Training Error = %.2f%%" % (training_error(X_loc, y_loc, clf_loc) * 100)
print "Storing Classifiers..."
pickle.dump((clf_person, clf_org, clf_loc), open("classifiers.dat", "w"))
def ml_ranker(entities, person_mat, org_mat, loc_mat):
clf_person, clf_org, clf_loc = pickle.load(open("classifiers.dat", "r"))
data_set = [(person_mat, clf_person), (org_mat, clf_org), (loc_mat, clf_loc)]
ret = []
scores = []
for data, clf in data_set:
normalize(data)
score = [0] * len(data)
for i in xrange(len(data) - 1):
for j in xrange(i + 1, len(data)):
if clf.predict(dist_vector(data[i], data[j])) == [1]:
score[i] += 1
else:
score[j] += 1
if clf.predict(dist_vector(data[j], data[i])) == [-1]:
score[i] += 1
else:
score[j] += 1
result = sorted(range(len(data)), key=lambda i: -score[i])
ret.append(result)
scores.append(score)
return ret, scores
def test(self):
if self.model_path[len(self.model_path) - 3:] != '.h5':
print('Error: incompatible model file')
exit()
model = load_model(filepath=self.model_path, custom_objects={'content_loss': content_loss})
if isfile(self.input_path):
image = data.imread(self.input_path)
image = set_image_alignment(image, 2)
if image.shape[0] >= HEIGHT_LIMITED or image.shape[1] >= WIDTH_LIMITED:
image_batch, batch_shape = split_for_test(image)
image_batch = normalize(image_batch)
image_generated_batch = model.predict([image_batch])
image_generated = build_image_from_batch(image_generated_batch, batch_shape)
else:
image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
image = normalize(image)
image_generated = model.predict([image])
image_generated = image_generated.reshape(
(image_generated.shape[1], image_generated.shape[2], image_generated.shape[3]))
image = denormalize(image_generated)
save_image(self.output_path, image)
def train(self, X, y, cv, test_set, cv_y, test_y):
self.clear()
col = self.config['input_size']
cv = np.reshape(cv, (len(cv)/col, col))
test_set = np.reshape(test_set, (len(test_set)/col, col))
m = 0
X, mu, sigma = normalize(X)
# save X, mu and sigma
np.savetxt('X.csv', X, delimiter=',')
np.savetxt('mu.csv', mu, delimiter=',')
np.savetxt('sigma.csv', sigma, delimiter=',')
self.nn_params = optimize.fmin_cg(f, self.nn_params, args=(self, X, y), maxiter=50,
fprime=fprime)
# save nn_parameters
hidden_size = self.config['hidden_size']
input_size = self.config['input_size']
num_labels = self.config['num_labels']
theta1 = self.nn_params[:((hidden_size) * (input_size + 1))].reshape(
(hidden_size, input_size + 1))
theta2 = self.nn_params[((hidden_size) * (input_size + 1)):].reshape(
(num_labels, hidden_size + 1))
np.savetxt('Theta1.csv', theta1, delimiter=',')
np.savetxt('Theta2.csv', theta2, delimiter=',')
#test the model
p = predict1(theta1, theta2, X)
_accuracy = accuracy(p, y)
l_fscores = list_of_fscores(p, y, num_labels)
fscore = total_fscore(l_fscores)
#test the model on cross validation set
fscores_cv = 0.0
accuracy_cv = 0.0
l_fscores_cv = 0.0
no = 0
X1, mu1, sigma1 = normalize(np.array(cv), mu, sigma)
p_cv = predict1(theta1, theta2, X1)
accuracy_cv = accuracy(p_cv, cv_y)
l_fscores_cv = list_of_fscores(p_cv, cv_y, num_labels)
fscores_cv = total_fscore(l_fscores_cv)
#test the model on test set
X_test, mu_test, sigma_test = normalize(np.array(test_set), mu, sigma)
p_test = predict1(theta1, theta2, X_test)
accuracy_test = accuracy(p_test, test_y)
l_fscores_test = list_of_fscores(p_test, test_y, num_labels)
fscore_test = total_fscore(l_fscores_test)
print 'here are the shizz results:'
print 'fscores in cross validation:'
print fscores_cv, ' ', accuracy_cv
print 'fscores in test set: ', fscore_test
print 'accuracy_test: ', accuracy_test
return self.nn_cfx(X, y, self.nn_params), self.nn_params, fscores_cv, fscore_test
# simpler format for building your arrays
x = concat(a_train, c_train)
print('concatenated x1 and x2')
samples, r, c = x.shape
x = np.reshape(x, (samples, r * c))
y = create_labels(len(a_train), len(c_train)).ravel()
print("Created training data and labels")
print("x shape: ", x.shape)
print("y shape: ", y.shape)
# normalize x
normalize(x)
print("Normalized x")
test_data = concat(a_test, c_test)
samples, r, c = test_data.shape
test_data = np.reshape(test_data, (samples, r * c))
test_data_labels = create_labels(len(a_test), len(c_test)).ravel()
print("Created testing data and labels")
normalize(test_data)
print("Normalized test data")
def kps(self): if not hasattr(self, '_kps'): self._kps = normalize(self.Kinv, self.kpus) return self._kps
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
return x
model = Classifier().to(device)
xy = np.loadtxt('faults.csv', delimiter=',') #1941
x_data = xy[:, :-7]
y_data = xy[:, -7:]
if use_normalize:
normalize(x_data, exclude_cols)
x_data = torch.FloatTensor(x_data).to(device)
y_data = torch.FloatTensor(y_data).to(device)
x_data, y_data = SuffleData(x_data, y_data, len(xy))
x_train = x_data[:training_len]
y_train = y_data[:training_len]
x_test = x_data[training_len:]
y_test = y_data[training_len:]
test_len = len(x_test)
optimizer = optim.Adam(model.parameters(), lr=0.0005)
from scipy import misc
import cv2, numpy as np
from Utils import reverse, normalize
import Utils
Utils.SET_GPU_MEM() #set GPU memory limit, Default: 0.5
# ============= read row image ================
rowImgname = "yourIamgeName.jpg".split(".")[0]
rowImg = misc.imread(rowImgname + ".jpg")
h, w, c = rowImg.shape
os.makedirs(rowImgname, exist_ok=True)
# ============= generative mask_nobg ================
model_nobg = load_model("models/yourModelName.h5")
fake_nobg = cv2.resize(reverse(model_nobg.predict(normalize(rowImg))), (w, h))
Utils.CLEAR_SESSION()
del model_nobg
# ============= clean mask_nobg ================
mask_nobg = cv2.medianBlur(
Utils.noisetool().deNoise1_er(Utils.masktool().binary(fake_nobg, 32)), 7)
# ============= bitwise and ================
nobg_rgb = cv2.bitwise_and(rowImg, rowImg, mask=mask_nobg)
nobg_final = np.concatenate([nobg_rgb, mask_nobg.reshape(h, w, 1)], axis=-1)
misc.imsave(rowImgname + "/" + rowImgname + "-gan.jpg", fake_nobg)
misc.imsave(rowImgname + "/" + rowImgname + "-mask.jpg", mask_nobg)
misc.imsave(rowImgname + "/" + rowImgname + "-nobg.png", nobg_final)
