def training_l(X, Y, X_test, Y_test, supp, c, s,
epochs=10, n_hidden1=1, learning_rate=0.01, Ts=1000, step=1, C=3.):
N, p = X.shape
torch.manual_seed(1) # set seed
# Define neural network model
model = Net_linear(n_feature=p, n_hidden1=n_hidden1, n_output=1)
# reserve model for the save of best model
best_model = Net_linear(n_feature=p, n_hidden1=n_hidden1, n_output=1)
# Define optimization algorithm for two sets of parameters, where weight_decay is the lambdas
optimizer = torch.optim.SGD(list(model.parameters()), lr=0.001, weight_decay=0.0025*c)
optimizer0 = torch.optim.SGD(model.hidden0.parameters(), lr=0.001, weight_decay=0.0005*c)
# Define loss function
lf = torch.nn.MSELoss()
# Take track of loss function values and supports over iteration
hist = []
SUPP = []
supp_x = range(p) # initial support
SUPP.append(supp_x)
### DFS algorithm
for i in range(epochs):
# One DFS epoch
model, supp_x, _ = DFS_epoch(model, s, supp_x, X, Y, lf, optimizer0, optimizer, Ts, step)
supp_x.sort()
# Save current loss function value and support
hist.append(lf(model(X), Y).data.numpy().tolist())
SUPP.append(supp_x)
# Prevent divergence of optimization over support, save the current best model
if hist[-1] == min(hist):
best_model.load_state_dict(model.state_dict())
best_supp = supp_x
# Early stop criteria
if len(SUPP[-1]) == len(SUPP[-2]) and len(set(SUPP[-1]).difference(SUPP[-2])) == 0:
break
# metrics calculation
fs = set(best_supp).difference(supp) # false selection number
ns = set(supp).difference(best_supp) # negative selection number
_err_train = mse(best_model, X, Y) # training error
_err_test = mse(best_model, X_test, Y_test) # testing error
_bic = N*np.log(_err_train) + C*s*np.log(N) # bic
### Second step training (for two-step procedure)
_optimizer = torch.optim.Adam(list(best_model.parameters())[1:], lr=0.5)
for _ in range(5000):
out = best_model(X)
loss = lf(out, Y)
_optimizer.zero_grad()
loss.backward()
_optimizer.step()
hist.append(loss.data.numpy().tolist())
bic = N*np.log(loss.data.numpy().tolist()) + C*s*np.log(N) # bic based on final model
err_train = mse(best_model, X, Y)
err_test = mse(best_model, X_test, Y_test)
return best_model, best_supp, bic, [_err_train, _err_test], [err_train, err_test]
def __init__(self, state_size, history_size=1, dropout_keep_prob=-1, embedding_size=-1, reuse=False):
self.state_size = state_size
with tf.variable_scope('Critic' if not reuse else "ShareLatent" , reuse=reuse):
self.inputs = tf.placeholder(tf.float32, [None, history_size, self.state_size])
self.returns = tf.placeholder(tf.float32, [None, ])
self.inputs_flat = tf.reshape(self.inputs, [-1, self.state_size * history_size])
if embedding_size != -1:
self.task_input = tf.placeholder(tf.float32, [None, embedding_size])
self.inputs_flat = tf.concat([self.task_input, self.inputs_flat], 1)
self.W_fc1 = _fc_weight_variable([self.state_size * history_size + embedding_size, HIDDEN_SIZE], name = "W_fc1")
else:
self.W_fc1 = _fc_weight_variable([self.state_size * history_size, HIDDEN_SIZE], name = "W_fc1")
self.b_fc1 = _fc_bias_variable([HIDDEN_SIZE], self.state_size, name = "b_fc1")
self.fc1 = tf.nn.relu(tf.nn.bias_add(tf.matmul(self.inputs_flat, self.W_fc1), self.b_fc1))
if dropout_keep_prob != -1:
self.fc1 = tf.nn.dropout(self.fc1, dropout_keep_prob)
with tf.variable_scope("Value"):
self.W_fc2 = _fc_weight_variable([HIDDEN_SIZE, 1], name = "W_fc3")
self.b_fc2 = _fc_bias_variable([1], HIDDEN_SIZE, name = "b_fc3")
self.value = tf.nn.bias_add(tf.matmul(self.fc1, self.W_fc2), self.b_fc2)
self.value_loss = tf.reduce_mean(mse(tf.squeeze(self.value), self.returns))
self.variables = [self.W_fc1, self.b_fc1, self.W_fc2, self.b_fc2]
def __init__(self, state_size):
self.state_size = state_size
with tf.variable_scope('Critic'):
self.inputs = tf.placeholder(tf.float32, [None, self.state_size])
self.returns = tf.placeholder(tf.float32, [
None,
])
self.W_fc1 = self._fc_weight_variable([self.state_size, 256],
name="W_fc1")
self.b_fc1 = self._fc_bias_variable([256],
self.state_size,
name="b_fc1")
self.fc1 = tf.nn.relu(
tf.matmul(self.inputs, self.W_fc1) + self.b_fc1)
with tf.variable_scope("Value"):
self.W_fc2 = self._fc_weight_variable([256, 1], name="W_fc3")
self.b_fc2 = self._fc_bias_variable([1], 256, name="b_fc3")
self.value = tf.matmul(self.fc1, self.W_fc2) + self.b_fc2
self.value_loss = tf.reduce_mean(
mse(tf.squeeze(self.value), self.returns))
self.variables = [self.W_fc1, self.b_fc1, self.W_fc2, self.b_fc2]
def main(p_miss=0.5,
hidden_size=100,
epochs=70,
lr=0.001,
dataset="drive",
mode="mcar",
para=0.5,
train=None):
n, p, xmiss, xhat_0, mask, data_x, data_y = load_data(p_miss,
dataset=dataset,
mode=mode,
para=para,
train=train)
imputer = MiceImputer(np.nan)
X = xmiss
X_filled, specs = imputer.transform(
np.array(X), (hidden_size, hidden_size, hidden_size),
epochs=epochs,
lr=lr,
iterations=10)
mse_nn = mse(X_filled, data_x, mask)
print("MSE MICE_NN : ", mse_nn)
return X_filled, mse_nn
def learn(self):
env = self.env
for episode in range(1, self.num_episodes + 1):
env.reset()
state1 = env.observe()
E = np.zeros_like(self.w)
while state1 != TERMINAL_STATE:
Qhat1, action1 = self.policy(state1)
state2, reward = env.step(action1)
Qhat2, action2 = self.policy(state2)
feats1 = phi(state1, action1)
grad_w_Qhat1 = feats1
delta = reward + self.gamma * Qhat2 - Qhat1
E = self.gamma * self.lmbd * E + grad_w_Qhat1
dw = self.alpha * delta * E
self.w += dw
state1 = state2
if self.save_error_history:
self.Q = expand_Q(self.w)
self.error_history.append((episode, mse(self.Q, self.opt_Q)))
self.Q = expand_Q(self.w)
return self.Q
def gtv_cvlam(X, y, q, num_folds=5, num_lams=20):
n = len(X)
folds = create_folds(n, num_folds)
scores = np.zeros(num_lams)
lams = None
for i, fold in enumerate(folds):
mask = np.ones(n, dtype=bool)
mask[fold] = False
x_train, y_train = X[mask], y[mask]
x_test, y_test = X[~mask], y[~mask]
data, weights, grid = bucket_vals(x_train, y_train, q)
results = solve_gfl(data,
None,
weights=weights,
full_path=True,
minlam=0.1,
maxlam=20.,
numlam=num_lams)
fold_score = np.array([
mse(y_test, predict(x_test, beta, grid))
for beta in results['beta']
])
scores += fold_score
if i == 0:
lams = results['lambda']
scores /= float(num_folds)
lam_best = lams[np.argmin(scores)]
data, weights, grid = bucket_vals(X, y, q)
beta = solve_gfl(data, None, weights=weights, lam=lam_best)
return beta.reshape(q), grid
def ml(ml_data, target, ml_keys, ml_score_keys):
import xgboost
global x_train_ibis_original
global y_train_ibis_original
global x_valid_ibis_original
global y_valid_ibis_original
ml_times = {key: 0.0 for key in ml_keys}
ml_scores = {key: 0.0 for key in ml_score_keys}
(x_train, y_train, x_test,
y_test), ml_times["t_train_test_split"] = split_step(ml_data, target)
if target == 'target0':
x_train_ibis_original = x_train
y_train_ibis_original = y_train
x_valid_ibis_original = x_test
y_valid_ibis_original = y_test
t0 = timer()
training_dmat_part = xgboost.DMatrix(data=x_train, label=y_train)
testing_dmat_part = xgboost.DMatrix(data=x_test, label=y_test)
ml_times["t_dmatrix"] = round((timer() - t0) * 1000)
watchlist = [(testing_dmat_part, "eval"), (training_dmat_part, "train")]
xgb_params = {
"objective": "binary:logistic",
"tree_method": "hist",
"max_depth": 1,
"nthread": 56,
"eta": 0.1,
"silent": 1,
"subsample": 0.5,
"colsample_bytree": 0.05,
"eval_metric": "auc",
}
t0 = timer()
model = xgboost.train(
xgb_params,
dtrain=training_dmat_part,
num_boost_round=10000,
evals=watchlist,
early_stopping_rounds=30,
maximize=True,
verbose_eval=1000,
)
ml_times["t_train"] = round((timer() - t0) * 1000)
t0 = timer()
yp = model.predict(testing_dmat_part)
ml_times["t_inference"] = round((timer() - t0) * 1000)
ml_scores["mse"] = mse(y_test, yp)
ml_scores["cod"] = cod(y_test, yp)
ml_times["t_ml"] += round(ml_times["t_train"] + ml_times["t_inference"])
return ml_scores, ml_times
def main():
mse_xs = []
mse_ys = []
q_star_sa = mc_control(1000000)
for lamba in range(0, 11):
for (_, q_sa) in sarsa_lambda(num_episodes=1000, lamba=lamba / 10):
mse_xs.append(lamba / 10)
mse_ys.append(mse(q_sa, q_star_sa))
mse_0_xs = []
mse_0_ys = []
for (n, q_sa) in sarsa_lambda(num_episodes=1000,
lamba=0,
yield_progress=True):
mse_0_xs.append(n)
mse_0_ys.append(mse(q_sa, q_star_sa))
mse_1_xs = []
mse_1_ys = []
for (n, q_sa) in sarsa_lambda(num_episodes=1000,
lamba=1,
yield_progress=True):
mse_1_xs.append(n)
mse_1_ys.append(mse(q_sa, q_star_sa))
fig, axarr = plt.subplots(1, 3)
ax0 = axarr[0]
ax0.plot(mse_xs, mse_ys)
ax0.axis([0, 1, 0, 1])
ax0.set_xlabel("lambda")
ax0.set_ylabel("MSE")
ax1 = axarr[1]
ax1.plot(mse_0_xs, mse_0_ys)
ax1.axis([0, 1000, 0, 1])
ax1.set_xlabel("Episode")
ax1.set_ylabel("MSE")
ax1.set_title("lambda=0")
ax2 = axarr[2]
ax2.plot(mse_1_xs, mse_1_ys)
ax2.axis([0, 1000, 0, 1])
ax2.set_xlabel("Episode")
ax2.set_ylabel("MSE")
ax2.set_title("lambda=1")
plt.show()
def ml(X, y, random_state, n_runs, test_size, optimizer, ml_keys,
ml_score_keys):
if optimizer == "intel":
print("Intel optimized sklearn is used")
from daal4py.sklearn.model_selection import train_test_split
import daal4py.sklearn.linear_model as lm
if optimizer == "stock":
print("Stock sklearn is used")
from sklearn.model_selection import train_test_split
import sklearn.linear_model as lm
else:
print(
f"Intel optimized and stock sklearn are supported. {optimizer} can't be recognized"
)
sys.exit(1)
clf = lm.Ridge()
mse_values, cod_values = [], []
ml_times = {key: 0.0 for key in ml_keys}
ml_scores = {key: 0.0 for key in ml_score_keys}
print("ML runs: ", n_runs)
for i in range(n_runs):
(X_train, y_train, X_test,
y_test), split_time = split(X,
y,
test_size=test_size,
random_state=random_state)
ml_times["t_train_test_split"] = split_time
random_state += 777
t0 = timer()
model = clf.fit(X_train, y_train)
ml_times["t_train"] += round((timer() - t0) * 1000)
t0 = timer()
y_pred = model.predict(X_test)
ml_times["t_inference"] += round((timer() - t0) * 1000)
mse_values.append(mse(y_test, y_pred))
cod_values.append(cod(y_test, y_pred))
ml_times["t_ml"] += ml_times["t_train"] + ml_times["t_inference"]
ml_scores["mse_mean"] = sum(mse_values) / len(mse_values)
ml_scores["cod_mean"] = sum(cod_values) / len(cod_values)
ml_scores["mse_dev"] = pow(
sum([(mse_value - ml_scores["mse_mean"])**2
for mse_value in mse_values]) / (len(mse_values) - 1),
0.5,
)
ml_scores["cod_dev"] = pow(
sum([(cod_value - ml_scores["cod_mean"])**2
for cod_value in cod_values]) / (len(cod_values) - 1),
0.5,
)
return ml_scores, ml_times
def test_gauss_filter(img):
print('\t[Loading...] Calculating..')
start = perf_counter()
result = gauss_filter(img)
finish = perf_counter()
cv2.imshow('Gauss Filter', result)
print('\tTime: ' + str(finish - start))
mse_value = mse(img, result)
print('\tMSE: ' + str(mse_value) + '\n')
def compute_dist_distribution(npy_path_list, average_image):
n = len(npy_path_list)
result = np.empty(n)
for i in range(n):
cur_img = np.load(npy_path_list[i])
result[i] = utils.mse(average_image, cur_img)
return result
def test_opencv_bilateral_filter(img):
print('\t[Loading...] Calculating..')
result = img.copy()
start = perf_counter()
cv2.bilateralFilter(result, 5, 50, 100)
finish = perf_counter()
cv2.imshow('OpenCV Bilateral Filter', result)
print('\tTime: ' + str(finish - start))
mse_value = mse(img, result)
print('\tMSE: ' + str(mse_value) + '\n')
def validate(self):
test_data = self.r_test
data = self.r_train
for u in self.ux:
for i in self.ix:
data[u, i] = self.predict(u, i)
print('Mean Squared Error: {}'.format(mse(data, test_data)))
def get_loss(model, placeholder_dict):
a = placeholder_dict["A"]
adv = placeholder_dict["ADV"]
r = placeholder_dict["R"]
# Compute cross entropy loss between estimated distribution of action and 'true' distribution of actions
chosen_action_log_probs = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=model.pi_logit, labels=a)
pg_loss = tf.reduce_mean(adv * chosen_action_log_probs) # minimize
vf_loss = tf.reduce_mean(mse(tf.squeeze(model.vf), r)) # minimize
entropy = -tf.reduce_mean(cat_entropy(model.pi_logit)) # maximize
return pg_loss, entropy, vf_loss, model.vf, chosen_action_log_probs, None, None
def adjust_weights(X, y, w, b, epochs, losses, lr, func_adjust):
for _ in range(epochs):
y_hat = dot(X, w) + b
dw, db = func_adjust(X, y, y_hat)
w -= lr * dw
b -= lr * db
error = mse(y, dot(X, w) + b)
losses.append(error)
return w, b, losses
def loss(self, x, y, training=True):
with tf.name_scope('loss'):
z = self._encode(x, training=training)
x_h = self._decode(z, training=training)
loss = dict()
#loss['pmse'] = p_mse(x, x_h)
#loss['corr'] = cc(x, x_h)
#loss['diff'] = l1(x, x_h)
loss['mse'] = mse(y, x_h)
#tf.summary.scalar('pmse', loss['pmse'])
#tf.summary.scalar('corr', loss['corr'])
#tf.summary.scalar('diff', loss['diff'])
tf.summary.scalar('mse', loss['mse'])
return loss
def test_canny(img):
start = perf_counter()
res = canny(img)
finish = perf_counter()
cv2.imshow('Canny', res)
print('Time = ' + str(finish - start))
start = perf_counter()
res_cv = cv2.Canny(img, 50, 70)
finish = perf_counter()
cv2.imshow('OpenCV Canny', res_cv)
print('Time_opencv = ' + str(finish - start))
mse_value = mse(res, res_cv)
print('MSE: ' + str(mse_value) + '\n')
def train(target_year):
print(target_year)
cache_path = './cache'
if os.path.exists(os.path.join(cache_path, "{}_boosted_tree.pkl".format(target_year))):
df = pickle.load(open(os.path.join(cache_path, "{}_boosted_tree.pkl".format(target_year)), 'rb'))
else:
df = load_data(target_year)
pickle.dump( df, open(os.path.join(cache_path, "{}_boosted_tree.pkl".format(target_year)), 'wb'))
selected_feature = const_selected_feature
X = df[selected_feature].values
bde = joblib.load('dictionary/category_encoder.pkl')
# bde = ce.BinaryEncoder(cols=categorical_feature, return_df=False)
venue_embedding = bde.transform(df[['venue']]).values
X = np.hstack((X, venue_embedding))
nlp_pipeline = joblib.load('dictionary/nlp_pipeline.pkl')
embeddings = nlp_pipeline.transform(df['summary'].values)
X = np.hstack((X, embeddings))
y = df['citationCount'].values
y = np.clip( y, 0, 41)
kf = KFold(n_splits=5)
accuracies = []
r2_correlations = []
for train_index, test_index in tqdm(kf.split(y)):
ensemble = build_model()
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
# ensemble.fit(X_train, y_train)
# y_pred = ensemble.predict(X_test)
y_pred = [np.mean(y_train)]*len(y_test)
mse_score = mse(y_pred, y_test)
r2 = r2_score(y_test, y_pred)
r2_correlations.append(r2)
accuracies.append(mse_score)
print("MSE: {} ({})".format(np.mean(accuracies), np.std(accuracies)))
print("R2 : {} ({})".format(np.mean(r2_correlations), np.std(r2_correlations)))
print("Sample : ")
print(y_pred[:10])
print(y_test[:10])
def getBestMatchIndex(input_image, boundingRect, subImage, arabicTemplates, yUps, yDowns):
bestError,result,index = 1000000,0,0
yUps = np.concatenate((np.zeros(10),yUps),axis=1)
yDowns = np.concatenate((np.zeros(10),yDowns),axis=1)
for image in arabicTemplates:
subImage = getSubImage(boundingRect ,input_image,yUp=yUps[index],yDown=yDowns[index])
subImage = cv2.cvtColor(subImage,cv2.COLOR_BGR2GRAY)
height,width = subImage.shape
image = cv2.resize(image,(width, height), interpolation = cv2.INTER_CUBIC)
image = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
#ret,image = cv2.threshold(image,127,255,0)
mseError = mse(subImage, image)
if(mseError < bestError):
bestError = mseError
result = index
index+=1
return result
def valid(self, epoch, val_loader):
print("\nStart Val")
self.G.eval()
mse_loss = 0
for i, (input_, target_, _) in enumerate(val_loader):
input_ = Variable(input_, volatile=True).cuda()
target_ = Variable(target_, volatile=True).cuda()
output_ = G(input_)
target_ = target_.cpu.data[0].numpy()
output_ = output_.cpu.data[0].numpy()
mse_loss += utils.mse(output_, target_)
mse_loss /= len(val_loader)
print("[Val] epoch:{} / best_mse:{}\n".format(epoch, best_mse))
if mse_loss < self.best_mse:
self.best_mse = mse_loss.data[0]
self.save()
print("End Val\n")
def eval_togtd_per_run(env, truth, stat_dist, runtime, runtimes, episodes,
target, behavior, gamma, Lambda, alpha, beta,
evaluation):
print('running %d of %d for togtd' % (runtime + 1, runtimes))
value_trace = true_online_gtd(env,
episodes,
target,
behavior,
Lambda,
gamma=gamma,
alpha=alpha,
beta=beta,
evaluation=evaluation)
if evaluation is not None:
return value_trace.T
else:
result = np.zeros((1, episodes))
for j in range(len(value_trace)):
result[0, j] = mse(value_trace[j], truth, stat_dist)
return result
def runTests(X, Y):
Xtr, Xte, Ytr, Yte = utils.splitData(X, Y, 0.9)
print("X and Y shapes =", X.shape, Y.shape)
results, estimator = nn.train(Xtr, Ytr)
print(results)
estimator.fit(Xtr, Ytr)
Yhat = estimator.predict(Xte)
mse = utils.mse(Yte, Yhat)
print("mse on testing data is", mse)
return (
results,
mse,
)
def adjust_weights_with_batch(X, y, w, b, epochs, batch, losses, lr, func_adjust):
n_rows, _ = X.shape
for _ in range(epochs):
for i in range((n_rows - 1) // batch + 1):
start_i = i * batch
end_i = start_i + batch
idx = list(range(start_i, end_i))
Xb = X[idx, :]
yb = y[idx, :]
y_hat = dot(Xb, w) + b
dw, db = func_adjust(Xb, yb, y_hat)
w -= lr * dw
b -= lr * db
error = mse(y, dot(X, w) + b)
losses.append(error)
return w, b, losses
def learn(self):
env = self.env
Q = self.Q
N = np.zeros(STATE_SPACE_SHAPE)
for episode in range(1, self.num_episodes + 1):
env.reset()
state1 = env.observe()
E = np.zeros(STATE_SPACE_SHAPE) # eligibility traces
while state1 != TERMINAL_STATE:
action1 = epsilon_greedy_policy(Q, N, state1)
state2, reward = env.step(action1)
dealer1, player1 = state1
idx1 = (dealer1 - 1, player1 - 1, action1)
Q1 = Q[idx1]
if state2 == TERMINAL_STATE:
Q2 = 0.0
else:
action2 = epsilon_greedy_policy(Q, N, state2)
dealer2, player2 = state2
idx2 = (dealer2 - 1, player2 - 1, action2)
Q2 = Q[idx2]
N[idx1] += 1
E[idx1] += 1
alpha = 1.0 / N[idx1]
delta = reward + self.gamma * Q2 - Q1
Q += alpha * delta * E
E *= self.gamma * self.lmbd
state1 = state2
if self.save_error_history:
self.error_history.append((episode, mse(self.Q, self.opt_Q)))
return Q
def mSDA_cv(p, x, y, n_cv=5):
kf = KFold(n_splits = n_cv)
res = np.zeros((p.size, n_cv))
for j, pj in enumerate(p):
i = 0
for train, test in kf.split(x):
x_temp, y_temp = x[train], y[train]
x_test, y_test = x[test], y[test]
fit_sda = mSDA(x_temp.T,pj,1)
x_sda = fit_sda[-1][-1].T
w_sda = fit_sda[0]
x_test_sda = mSDA_features(w_sda, x_test.T).T
lr_sda = linear_model.LinearRegression()
lr_sda.fit(x_sda, y_temp)
res[j,i] = utils.mse(lr_sda, x_test_sda, y_test)
i += 1
res = np.mean(res,1)
return p[np.argmin(res)]
def learn(self, actions=actions, terminal=terminal):
N = np.zeros(self.state_space)
Q = np.zeros(self.state_space)
if self.save_error == True:
self.error = []
for _ in range(0, self.episodes):
game = self.env()
state = game.state
E = np.zeros(self.state_space)
action = random.choice(actions)
for _ in range(0, self.max_steps):
state_prime, reward = game.step(action)
player, dealer = state
index = player - 1, dealer - 1, action
N[index] += 1
if state_prime == terminal:
td_error = reward - Q[index]
else:
action_prime = ep_greedy(N, Q, state_prime, self.N_0)
player_prime, dealer_prime = state_prime
index_prime = player_prime - 1, dealer_prime - 1, action_prime
td_error = reward + (Q[index_prime] - Q[index])
E[index] += 1
alpha = 1 / N[index]
Q += alpha * td_error * E
E *= self.lmbda * self.gamma
if state_prime == terminal:
break
state, action = state_prime, action_prime
if self.save_error == True:
mse_ep = mse(Q, self.q_star)
self.error.append(mse_ep)
if self.save_error == True:
return (Q, self.error)
else:
return (Q)
def test_mse_gives_0(self):
x = np.array([0, 0])
positions = [np.array([1, 0])]
distances = [1]
self.assertEqual(utils.mse(x, positions, distances), 0)
def test_mse_gives_non_zero_for_multiple_beacons(self):
x = np.array([0, 0])
positions = [np.array([1, 0]), np.array([1, 0])]
distances = [1.5, 2]
self.assertEqual(utils.mse(x, positions, distances), 1.25 / 2)
def computeScore(self,dart_image):
if(self.boardImage is None):
return False,False
else:
### set output image
self.outputBoardImage = self.boardImage.copy()
###scale dart image
self.dartImage = transform.rescale(dart_image, 0.8, anti_aliasing=True,multichannel=True)
self.dartImage = skimage.img_as_ubyte(self.dartImage)
### crop dart image
self.dartImage = utils.crop_image(self.dartImage)
if(self.dartImage is None):
return False,False
### align the two images using SIFT TECHNIQUE
self.dartImage = utils.alignImages(self.dartImage,self.boardImage)
###### Difference of 2 Images
diff = utils.mse(self.boardImage,self.dartImage)
if(diff == 0):### the two images are the same
return False,False
diff_image = utils.computeDifference(cv2.cvtColor(self.dartImage,cv2.COLOR_RGB2GRAY),cv2.cvtColor(self.boardImage,cv2.COLOR_RGB2GRAY))
dart = np.multiply(diff_image,self.myMask.board)
####Find primary orientation of largest difference region
[rows, columns, channels] = self.boardImage.shape
dart_square = np.power(dart,2)
SE = disk(np.round(rows/100))
dart_square_threshed = dart_square > 0.2 * threshold_otsu(dart_square)
self.myMask.dart = binary_dilation(dart_square_threshed,selem=SE)
label_img = label(self.myMask.dart)
region = regionprops(label_image=label_img)
max_index = utils.get_max_index(region)
#get the oriention in degrees
orientation = region[max_index].orientation
orientation = np.degrees(orientation)-90 # -90 to adjust the orientation
#get the line structure elemet with orientation
SE_line = utils.strel_line(length=50,degrees=orientation)
#close using the line to get the true shape of dart touching the board
self.myMask.dart = closing(dart_square_threshed,selem=SE_line)
#get the contours of the dart then detect the extrem left point which is the point the dart touch the board in
gray_dart = skimage.img_as_ubyte(self.myMask.dart)
cnts = cv2.findContours(gray_dart, cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
c = max(cnts, key=cv2.contourArea)
extLeft = tuple(c[c[:, :, 0].argmin()][0])
### compare the xhit and yhit of the dart with board to know which region the dart hit
self.dartScore = self.find_hitRegion(xhit=extLeft[0],yhit=extLeft[1],center=self.center,region=self.regions_details)
### draw a contour of the hit region
hit_region = skimage.img_as_ubyte(self.myMask.hit)
cnts = cv2.findContours(hit_region, cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
cv2.drawContours(self.outputBoardImage, cnts, -1, (50, 255, 50), 5)
return self.dartScore ,self.outputBoardImage
def pandas_original():
import pandas as pd
import xgboost as xgb
global x_train_pandas_original
global y_train_pandas_original
global x_valid_pandas_original
global y_valid_pandas_original
PATH = '/localdisk/benchmark_datasets/santander'
train_pd = pd.read_csv('%s/train.csv' % PATH)
for i in range(200):
col = 'var_%d' % i
var_count = train_pd.groupby(col).agg({col: 'count'})
var_count.columns = ['%s_count' % col]
var_count = var_count.reset_index()
train_pd = train_pd.merge(var_count, on=col, how='left')
for i in range(200):
col = 'var_%d' % i
mask = train_pd['%s_count' % col] > 1
train_pd.loc[mask, '%s_gt1' % col] = train_pd.loc[mask, col]
# train, test data split
train, valid = train_pd[:-10000], train_pd[-10000:]
x_train_pandas_original = train.drop(['target', 'ID_code'], axis=1)
y_train_pandas_original = train['target']
x_valid_pandas_original = valid.drop(['target', 'ID_code'], axis=1)
y_valid_pandas_original = valid['target']
xgb_params = {
'objective': 'binary:logistic',
'tree_method': 'hist',
'max_depth': 1,
'nthread': 56,
'eta': 0.1,
'silent': 1,
'subsample': 0.5,
'colsample_bytree': 0.05,
'eval_metric': 'auc',
}
dtrain = xgb.DMatrix(data=x_train_pandas_original,
label=y_train_pandas_original)
dvalid = xgb.DMatrix(data=x_valid_pandas_original,
label=y_valid_pandas_original)
watchlist = [(dvalid, 'eval'), (dtrain, 'train')]
clf = xgb.train(xgb_params,
dtrain=dtrain,
num_boost_round=10000,
evals=watchlist,
early_stopping_rounds=30,
maximize=True,
verbose_eval=1000)
yp = clf.predict(dvalid)
score_mse = mse(y_valid_pandas_original, yp)
score_cod = cod(y_valid_pandas_original, yp)
print('[pandas_original] Scores: ')
print(' mse = ', score_mse)
print(' cod = ', score_cod)
