def image_formation():
Ncopy = 1000
use_sPCA = False
image_idx = 0
SNR = .1
r_max = 64
seed = 0
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
Rl = np.expand_dims(np.exp(1j * 2 * np.pi * 45 / 360 * Freqs), axis=-1)
Coeff_rotated = Coeff_raw * Rl
image_rotated = recover_image(Coeff_rotated, Phi_ns, Freqs, r_max, Mean)
image_rotated_noisy = recover_image(np.expand_dims(Coeff[:, 0], axis=-1),
Phi_ns, Freqs, r_max, Mean)
plt.subplot(131)
plt.imshow(image0, cmap='gray')
plt.axis('off')
plt.subplot(132)
plt.imshow(image_rotated, cmap='gray')
plt.axis('off')
plt.subplot(133)
plt.imshow(image_rotated_noisy, cmap='gray')
plt.axis('off')
plt.savefig('mra_2d_observations_model.png')
plt.savefig('mra_2d_observations_model.eps')
plt.clf()
def draw_from_prior():
Ncopy = 10000
use_sPCA = True
image_idx = 0
SNR = 10
filename = 'E70s.mat'
r_max = 64
seed = 0
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
data_prior = np.expand_dims(4 * np.exp(-np.asarray(Freqs) / 8), axis=-1)
Gamma_a = np.diag(np.abs(data_prior[:, 0])**2)
np.random.seed(10)
for i in range(9):
x = 1 / np.sqrt(2) * (
np.random.multivariate_normal(np.zeros(
(Freqs.shape[0], )), Gamma_a) +
1j * np.random.multivariate_normal(np.zeros(
(Freqs.shape[0], )), Gamma_a))
Coeff_raw = np.expand_dims(x, axis=-1)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
plt.subplot('33%d' % (i + 1))
plt.imshow(image0, cmap='gray')
plt.axis('off')
plt.savefig('draw_from_prior.png')
plt.savefig('draw_from_prior.eps')
plt.clf()
def train(self):
#data agumentation
aug = ImageDataGenerator(rotation_range=25,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode="nearest")
# Model Checkpoint
cpt_save = ModelCheckpoint('models_save/my_weight.h5',
save_best_only=True,
monitor='val_acc',
mode='max')
X_data, y_data = make_data()
(X_train, X_val, y_train, y_val) = train_test_split(X_data,
y_data,
test_size=0.2,
random_state=123)
print("Training......")
self.model.fit(aug.flow(X_train, y_train, batch_size=32),
validation_data=(X_val, y_val),
callbacks=[cpt_save],
verbose=1,
epochs=50,
steps_per_epoch=len(X_train) / 32)
#lưu model sau khi đã trên xong
self.model.save('models_save/my_model.h5')
def run_test():
print('test')
pool = Pool(processes=cpu_count())
X, Y = make_data(pool, 'ted_7_ErasePunc_FullKorean__test.txt')
print('make test data end.')
X = norm_many(pool, X)
print('norm_data end.')
interference(X, Y, 'model.tfl')
def run_train(train_file):
print('train')
pool = Pool(processes=cpu_count())
X, Y = make_data(pool, train_file)
print('make train data end.')
X = norm_many(pool, X)
print('norm_data end.')
train(X, Y, 'model_MDM001.tfl')
def run_train():
pool = Pool(processes=cpu_count())
X, Y = make_data(pool, 'ted_7_ErasePunc_FullKorean__train.txt')
print('make train data end.')
X = norm_many(pool, X)
print('norm train data end.')
train(X, Y, 'model.tfl')
def ppm_em_experiment(seed, image_idx, SNR, Ncopy, Ndir, use_sPCA,
use_signal_prior, gamma, BW, path_learning, P):
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
TT = time()
Coeff_s, est_rotations = synchronize_2d(Coeff, Freqs, Ndir)
t_synch = time() - TT
x_s = np.expand_dims(np.mean(Coeff_s, axis=-1), axis=-1)
x = align_image(x_s, Coeff_raw, Freqs)
imager = recover_image(x, Phi_ns, Freqs, r_max, Mean)
err_synch = np.sqrt(
np.sum(np.sum((image0 - imager)**2, axis=-1), axis=0) /
np.sum(np.sum(image0**2, axis=-1), axis=0))
x, num_iter_synch_em, t_synch_em = EM_General_Prior(Coeff,
Freqs,
sigma,
Ndir,
1 / Ndir * np.ones(
(Ndir, 1)),
Ndir,
0,
Coeff_raw,
use_signal_prior,
uniform=False)
x = align_image(x, Coeff_raw, Freqs)
imager = recover_image(x, Phi_ns, Freqs, r_max, Mean)
err_synch_em = np.sqrt(
np.sum(np.sum((image0 - imager)**2, axis=-1), axis=0) /
np.sum(np.sum(image0**2, axis=-1), axis=0))
results = pd.DataFrame()
results = results.append(
{
'use_signal_prior': use_signal_prior,
'use_sPCA': use_sPCA,
'seed': seed,
'SNR': SNR,
'sigma': sigma,
'N': Ncopy,
'L': Coeff.shape[0],
'image_idx': image_idx,
'err': err_synch_em,
'num_iter': num_iter_synch_em,
'BW': BW,
'gamma': gamma,
't': t_synch + t_synch_em,
't_synch': t_synch,
't_em': t_synch_em,
'err_synch': err_synch
},
ignore_index=True)
return results
def run_test():
pool = Pool(processes=cpu_count())
X, Y = make_data(pool, 'ted_7_ErasePunc_FullKorean__test.txt')
print('make test data end.')
X = norm_many(pool, X)
print('norm test data end.')
pred = test(X, Y, 'model.tfl')
print('pred[:10]={}'.format(pred[:10]))
def load_example():
X, y = make_data(request.json['example'])
example_data = {
"X": X.tolist(),
"y": y.tolist()
}
id = db.data.insert(example_data)
return jsonify({
"id": str(id)
})
def known_rotations_experiment(seed, image_idx, SNR, Ncopy, Ndir, use_sPCA):
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
est_rotations = rotations
Coeff_s = np.zeros_like(Coeff)
for i in range(Ncopy):
Coeff_s[:, i] = Coeff[:, i] * np.exp(
-1j * 2 * np.pi * est_rotations[i] / 360 * Freqs)
t_synch = 0
h, bin_edges = np.histogram((est_rotations - rotations) % 360,
bins=np.arange(-0.5, 360 + 0.5))
measured_rho = np.expand_dims(h / np.sum(h), axis=-1)
x_s = np.expand_dims(np.mean(Coeff_s, axis=-1), axis=-1)
x = align_image(x_s, Coeff_raw, Freqs)
imager = recover_image(x, Phi_ns, Freqs, r_max, Mean)
err_synch = np.sqrt(
np.sum(np.sum((image0 - imager)**2, axis=-1), axis=0) /
np.sum(np.sum(image0**2, axis=-1), axis=0))
print('#### oracle ####')
print('e = %f' % err_synch)
results = pd.DataFrame()
results = results.append(
{
'use_sPCA': use_sPCA,
'seed': seed,
'SNR': SNR,
'sigma': sigma,
'N': Ncopy,
'L': Coeff.shape[0],
'image_idx': image_idx,
'rho': measured_rho,
'err': err_synch,
'num_iter': 0,
't': t_synch
},
ignore_index=True)
return results
def synchronize_and_match_experiment(seed, image_idx, SNR, Ncopy, Ndir,
use_sPCA, P):
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
TT = time()
Coeff_s, est_rotations = synchronize_and_match_2d(Coeff,
Freqs,
P=P,
L=Ndir)
t_synch = time() - TT
h, bin_edges = np.histogram((est_rotations - rotations) % 360,
bins=np.arange(-0.5, 360 + 0.5))
measured_rho = np.expand_dims(h / np.sum(h), axis=-1)
x_s = np.expand_dims(np.mean(Coeff_s, axis=-1), axis=-1)
x = align_image(x_s, Coeff_raw, Freqs)
imager = recover_image(x, Phi_ns, Freqs, r_max, Mean)
err_synch = np.sqrt(
np.sum(np.sum((image0 - imager)**2, axis=-1), axis=0) /
np.sum(np.sum(image0**2, axis=-1), axis=0))
print('#### synchronize and match ####')
print('e = %f' % err_synch)
results = pd.DataFrame()
results = results.append(
{
'use_sPCA': use_sPCA,
'seed': seed,
'SNR': SNR,
'sigma': sigma,
'N': Ncopy,
'L': Coeff.shape[0],
'image_idx': image_idx,
'rho': measured_rho,
'err': err_synch,
'num_iter': 0,
't': t_synch
},
ignore_index=True)
return results
def standard_em_experiment(seed, image_idx, SNR, Ncopy, Ndir, use_sPCA,
use_signal_prior):
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy, SNR=SNR, seed=seed, image_idx=image_idx, sPCA=use_sPCA)
image0 = recover_image(Coeff_raw, Phi_ns, Freqs, r_max, Mean)
x, num_iter_em_uniform, t_em_uniform = EM_General_Prior(Coeff,
Freqs,
sigma,
Ndir,
1 / Ndir * np.ones(
(Ndir, 1)),
Ndir,
0,
Coeff_raw,
use_signal_prior,
uniform=True)
x = align_image(x, Coeff_raw, Freqs)
imager = recover_image(x, Phi_ns, Freqs, r_max, Mean)
err_em_uniform = np.sqrt(
np.sum(np.sum((image0 - imager)**2, axis=-1), axis=0) /
np.sum(np.sum(image0**2, axis=-1), axis=0))
results = pd.DataFrame()
results = results.append(
{
'use_signal_prior': use_signal_prior,
'use_sPCA': use_sPCA,
'seed': seed,
'SNR': SNR,
'sigma': sigma,
'N': Ncopy,
'L': Coeff.shape[0],
'image_idx': image_idx,
'err': err_em_uniform,
'num_iter': num_iter_em_uniform,
't': t_em_uniform
},
ignore_index=True)
return results
def main(args):
#######################
# ディレクトリパスの読み込み
#######################
# 今日の日付
today_time = dt.datetime.today().strftime("%Y_%m_%d")
# 各ディレクトリの設定
DATA_DIR, SAVE_DIR = \
args.data_dir, args.save_dir
# 'SAVE_DIR'に今日の日付を足す
do_time = os.datetime.today().strftime("%H_%M_%S")
SAVE_DIR = os.path.join(SAVE_DIR, today_time, do_time)
if not os.path.isdir(SAVE_DIR):
os.makedirs(SAVE_DIR)
#######################
# Hyper Parameter
#######################
param_dic = {}
config = configparser.ConfigParser()
config.read('config.ini')
param_sphere = config['sphere']
level = param_dic['level'] = int(param_sphere['level'])
hemisphere = param_dic['hemisphere'] int(param_sphere['hemisphere'])
Q = param_dic['Q'] = 2**level
title = param_dic['title'] = param_sphere['title']
param = config['parameter']
batch_size = param_dic['batch_size'] = param['batch_size']
max_epochs = param_dic['max_epochs'] = param['max_epochs']
evaluate_num = param_dic['evaluate_sample_num_per_epoch'] = param['evaluate_num']
out_num = param_dic['out_num'] = param['out_num']
filter_num = param_dic['filter_num'] = param['filter_num']
filter_size = param_dic['filter_size'] = param['filter_size']
padding = param_dic['padding'] = param['padding']
stride = param_dic['stride'] = param['stride']
hidden_size = param_dic['hidden_size'] = param['hidden_size']
optimizer = param_dic['optimizer'] = param['optimizer']
lr = param_dic['lr'] = param['optimizer']
weight_init_std = param_dic['weight_init_std'] = param['weight_init_std']
# 保存
with open(SAVE_DIR + 'setting.txt', 'a') as f:
for key, value in param_dic.items():
f.write(str(key) + ':' + str(value))
f.write('\n')
#######################
# 球面座標の獲得
#######################
img_ori = np.zeros((224, 224, 3))
img_fish, pos = fish.make_considered_picture(img=img_ori, level=level, return_array=1)
# print("level: ", level)
# print(img_fish.shape)
#######################
# Load Data
#######################
ndar = np.load(data_dir)
train = ndar['train']
test = ndar['test']
#######################
# Make data
# データをネットワークに適した形に変形する
#######################
x_train, y_train = make_data(train)
x_test, y_test = make_data(test)
#######################
# 球面画像の獲得
#######################
x_train = get_sphere_array(x_train, pos)
x_test = get_sphere_array(x_test, pos)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
if x_train.shape[-1] == 3:
print('change channel')
x_train = x_train[:, :, :, 0]
x_test = x_test[:, :, :, 0]
# if len(x_test) > 1000:
# tmp = 1000
# x_train = np.concatenate((x_train, x_test[tmp:]), axis=0)
# x_test = x_test[:tmp]
# y_train = np.concatenate((y_train, y_test[tmp:]), axis=0)
# y_test = y_test[:tmp]
x_test = x_test[:1000]
y_test = y_test[:1000]
#######################
# Train
#######################
network = SphereConv(input_dim=(3, 6, Q+2, 2*Q+2),
conv_param={'filter_num': filter_num,
'filter_size': filter_size,
'pad': padding, 'stride': stride},
hidden_size=hidden_size, output_size=out_num,
weight_init_std=weight_init_std, level=level)
trainer = Trainer(network, x_train, y_train, x_test[:100], y_test[:100],
epochs=max_epochs, mini_batch_size=batch_size,
optimizer=optimizer, optimizer_param={'lr': lr},
evaluate_sample_num_per_epoch=evaluate_num)
trainer.train()
#######################
# パラメータの保存
#######################
network.save_params(save_dir+"params.pkl")
print("Saved Network Parameters!")
try:
print('save acc')
a = np.array(trainer.train_acc_list)
b = np.array(train_acc_list.test_acc_list)
np.savez('acc.npz', train=a, test=b)
except:
print('error in saving acc')
# グラフの描画
markers = {'train': 'o', 'test': 's'}
x = np.arange(max_epochs)
plt.plot(x, trainer.train_acc_list, marker='o', label='train', markevery=2)
plt.plot(x, trainer.test_acc_list, marker='s', label='test', markevery=2)
plt.xlabel("epochs")
plt.ylabel("accuracy")
plt.ylim(0, 1.0)
plt.legend(loc='lower right')
plt.savefig(save_dir+"test.png")
label=r'$\sqrt{2}/2$')
plt.xlim(0, 0.5 * fs)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Gain')
plt.grid(alpha=0.5)
plt.legend(framealpha=1, shadow=True, loc='best')
plt.title("Amplitude response for\nButterworth bandpass filters", fontsize=10)
plt.text(430,
0.07,
"lowcut: %4g Hz\nhighcut: %4g Hz" % (lowcut, highcut),
fontsize=8)
plt.tight_layout()
plt.savefig("bandpass_example_response.pdf")
T = 0.03
t, x = make_data(T, fs)
y = butter_bandpass_filt(x, lowcut, highcut, fs, order=12)
plt.figure(figsize=(4.0, 2.8))
plt.plot(t, x, 'k', alpha=0.4, linewidth=1, label='Noisy signal')
plt.plot(t, y, 'C0', label='Filtered signal')
plt.xlabel('Time (seconds)')
plt.grid(alpha=0.5)
plt.axis('tight')
plt.xlim(0, T)
plt.legend(framealpha=1, shadow=True, loc='upper left')
plt.tight_layout()
plt.savefig("bandpass_example_signals.pdf")
def train (training_x, training_y, testing_x, testing_y):
lr=1e-4
decay=1e-6
epochs=10
model = get_model()
opt = Adam(lr=lr, decay=decay)
model.compile(loss='mse', optimizer=opt, metrics=['accuracy'])
model.fit(training_x, training_y, epochs=epochs, validation_data=(testing_x, testing_y))
return model
if __name__ == '__main__':
(training_x, training_y), (testing_x, testing_y) = make_data()
os.chdir("../../..")
trained_model = train(training_x, training_y, testing_x, testing_y)
#TODO: Save only the model and weights, not optimizer state or any of that junk
"""trained_model.save("./model.h5")
trained_model.save_weights('./model_weights.h5')
with open("./model.json", 'w') as file:
file.write(model.to_json())"""
trained_model.save("./tf-model", save_format="tf")
import tensorflow as tf
model_tf = tf.keras.models.load_model("./model.h5")
from make_data import make_data
def etccv(n_estimators, min_samples_split):
return cross_val_score(AdaBoostClassifier(ETC(min_samples_split=int(min_samples_split),
random_state=2,
n_jobs=-1),
algorithm="SAMME",
n_estimators=int(n_estimators)),
train, train_labels, 'roc_auc', cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path ="../input/xtrain_v5_full.csv",
test_path="../input/xtest_v5.csv")
# RF
etcBO = BayesianOptimization(etccv, {'n_estimators': (200, 800),
'min_samples_split': (2, 8)})
print('-' * 53)
etcBO.maximize()
print('-' * 53)
print('Final Results')
print('ETC: %f' % etcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(ETC(n_estimators=int(etcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(etcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
def load_model(self, path):
"""
:param path: path to model for loading
Stores saved parameters in self.w
"""
parameters = np.load(path, allow_pickle=True)
self.w = parameters
if __name__ == '__main__':
# Generating data
n = 1000
example_nr = 2
noise = 1
X, T, x, dim = make_data(example_nr, n, noise)
# Centering around Origo
X = X - np.mean(X, axis=0)
# Training settings
epochs = 100
batch_size = 64
learning_rate = 0.1
model_saving = {
'save': {
'do': True,
'path': '2_10_10_2_epoch100.npy'
},
'load': {
'do': False,
from plot_corr_normalized import plot_corr_normalized
from meta_data import *
from util_files import read_fit_file
import defines as df
import define_prior as dfp
import gvar as gv
import gvar.dataset as gvd
import matplotlib.pyplot as plt
import numpy as np
import sn_minimizer as snm
import sys
makeData = df.do_makedata_3pt
## -- for raw correlator file input
data,dset = make_data(df.mdp,do_makedata=makeData,\
do_db=False,filename="./import-correlators-bar3pt")
## --
inPrefix='v4v4'
inPostfix='t6'
outPrefix='prod3'
tsep=6
cmat = list()
if df.do_irrep == "8":
op_list = [1,2,3,5,6]
nameStr = "8p"
elif df.do_irrep == "8'":
op_list = [4,7]
nameStr = "8m"
elif df.do_irrep == "16":
def rfccv(n_estimators, min_samples_split):
return cross_val_score(RFC(n_estimators=int(n_estimators),
min_samples_split=int(min_samples_split),
random_state=2,
n_jobs=-1),
train,
train_labels,
'roc_auc',
cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = make_data(
train_path="../input/train.csv", test_path="../input/test.csv")
# RF
rfcBO = BayesianOptimization(rfccv, {
'n_estimators': (600, 800),
'min_samples_split': (2, 5)
})
print('-' * 53)
rfcBO.maximize()
print('-' * 53)
print('Final Results')
print('RFC: %f' % rfcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(RFC(
n_estimators=int(rfcBO.res['max']['max_params']['n_estimators']),
#for file_name in os.listdir('pokerdata'):
for i in xrange(1):
file_name = 'vali.txt'
with open(file_name) as f:
test_file = f.read()
if '9-max Seat' in test_file:
continue
test_file = test_file.replace('\xe2\x82\xac', '$')
games = re.findall(r'PokerStars Zoom Hand \#.+?FLOP.+?\*\*\* SUMMARY \*\*\*', test_file, re.DOTALL)
for game in games:
if not 'Seat 6' in game:
continue
c += 1
if c % 10 == 0:
print c, '/', len(games)
game_driver = make_data(game, sys.argv[2])
game_driver.game_stream(-1)
else:
with open(sys.argv[1]) as f:
test_file = f.read()
games = re.findall(r'PokerStars Zoom Hand \#.+?PokerStars Zoom Hand', test_file, re.DOTALL)
print len(games)
random.shuffle(games)
for game in games:
while True:
if re.findall(r'deoxy1909.*folded before Flop', game):
break
game = '\n'.join(game.split('\n')[:-3])
change_terminal_color('green')
print game.strip('\n')
# del_stdout_line(len(game.splitlines())+1)
def learn_signal_prior():
Ncopy = 100
SNR = 0.1
seed = 1
image_idx = 5
use_sPCA = True
r_max = 64
Freqs_t = np.zeros((0, ))
Coeff_raw_t = np.zeros((0, 1))
for image_idx in range(20):
Coeff, Freqs, rad_freqs, Mean, Phi_ns, sigma, Coeff_raw, rotations = make_data(
Ncopy=Ncopy,
SNR=SNR,
seed=seed,
image_idx=image_idx,
sPCA=use_sPCA)
Freqs_t = np.concatenate([Freqs_t, Freqs], axis=0)
Coeff_raw_t = np.concatenate([Coeff_raw_t, Coeff_raw], axis=0)
Freqs = Freqs_t
Coeff_raw = Coeff_raw_t
plt.subplot(211)
plt.scatter(Freqs, np.abs(Coeff_raw), alpha=.5, marker='+')
loss_vec = []
x = np.expand_dims(Freqs, axis=-1)
y = np.abs(Coeff_raw)
alpha_range = np.arange(1 / 32, 0.3, 1 / 32)
# alpha_range = [0,0.125]
for alpha in alpha_range:
x0 = 0
A = np.concatenate([np.exp(-alpha * (x - x0))], axis=-1)
w = np.linalg.pinv(A) @ y
loss = np.mean((y - w[0] * np.exp(-alpha * (x - x0)))**2)
loss_vec.append(loss)
#plt.plot(w[0] * np.exp(-alpha * (np.arange(np.min(Freqs), np.max(Freqs)) - x0)), 'k', alpha=0.1)
alpha_star = alpha_range[np.argmin(loss_vec)]
A = np.concatenate([np.exp(-alpha_star * (x - x0))], axis=-1)
w_opt = np.linalg.pinv(A) @ y
plt.plot(w[0] * np.exp(-alpha_star *
(np.arange(np.min(Freqs), np.max(Freqs)) - x0)),
'r',
alpha=0.5)
plt.xlabel('Angular Frequency')
plt.ylabel('Absolute Coefficient')
plt.text(75,
1,
r'$C=%.2f e^{%.3f \cdot f}$' % (w_opt, alpha_star),
bbox=dict(facecolor='yellow', alpha=0.5))
plt.subplot(212)
plt.plot(alpha_range, loss_vec)
plt.xlabel(r'$\alpha$')
plt.ylabel('loss')
plt.text(alpha_star - 0.025,
np.mean(loss_vec),
r'$\alpha^*=%.3f$' % alpha_star,
bbox=dict(facecolor='yellow', alpha=0.5))
plt.tight_layout()
plt.savefig('2d_figures/fit_signal_prior_.png', bbox_inches="tight")
plt.savefig('2d_figures/fit_signal_prior_.eps', bbox_inches="tight")
from sklearn.ensemble import RandomForestClassifier as RFC
from bayesian_optimization import BayesianOptimization
from make_data import make_data
def rfccv(n_estimators, min_samples_split):
return cross_val_score(RFC(n_estimators=int(n_estimators),
min_samples_split=int(min_samples_split),
random_state=2,
n_jobs=-1),
train, train_labels, 'roc_auc', cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = make_data(train_path = "../input/train.csv", test_path="../input/test.csv")
# RF
rfcBO = BayesianOptimization(rfccv, {'n_estimators': (600, 800),
'min_samples_split': (2, 5)})
print('-' * 53)
rfcBO.maximize()
print('-' * 53)
print('Final Results')
print('RFC: %f' % rfcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(RFC(n_estimators=int(rfcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(rfcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
criterion="gini",
min_samples_leaf=3,
max_depth=15,
max_features="auto",
oob_score=True,
random_state=23)
# Columns of output dataframe
DF_COLS = [
"n_samples", "n_features", "n_informative", "n_redundant", "n_repeated",
"n_classes", "n_clusters_per_class", "seed", "mr", "n_diff"
]
all_data = make_data(N_SAMPLES,
N_CLASSES,
N_FEATURES,
N_INFO,
N_PER,
use_seed=True)
def run(idx):
print("--Running Job for Index {}--".format(idx))
# Generate Data
data = all_data[idx]["data"]
train_x, train_y = data["train"]
test_x, test_y = data["test"]
# Create, fit, and predict a RFC
initial_clf = RandomForestClassifier(
from plot_corr_effective_mass import plot_corr_effective_mass
from plot_corr_normalized import plot_corr_normalized
from meta_data import *
from util_files import read_fit_file
import defines as df
import define_prior as dfp
import gvar as gv
import gvar.dataset as gvd
import matplotlib.pyplot as plt
import numpy as np
import sys
if df.do_db:
## -- for database input
## - (database file name defined in make_data_db.py)
data,dset = make_data(df.mdp,do_makedata=df.do_makedata,do_db=True)
models = make_models(data=data,lkey=df.lkey)
prior = make_prior(models)
else:
## -- for raw correlator file input
data,dset = make_data(df.mdp,do_makedata=df.do_makedata,\
do_db=False,filename="./import-correlators-bar2pt")
data3,dset3 = make_data(df.mdp,do_makedata=df.do_makedata_3pt,\
do_db=False,filename="./import-correlators-bar3pt")
models = make_models(data=data,lkey=df.lkey)
prior = make_prior(models)
## --
## -- DEPRICATED
#if df.do_uncorr:
# ## -- remove the correlations from the data
import multiprocessing as mp
# Imports from own modules
from make_data import make_data
# ***** Start here *****
pool = mp.Pool(processes=(mp.cpu_count() - 1))
skf = StratifiedKFold(n_splits=5, random_state=42)
scaler = StandardScaler()
np.random.seed(42)
# %% Import transformed Data
X_train, y_train, X_test, y_test = make_data()
# %% Playing with ColumnTransformer and PipeLines
class ModelTransformer(BaseEstimator, TransformerMixin):
def __init__(self, model, name):
self.model = model
self.name = name
self.features = None
def fit(self, *args, **kwargs):
self.model.fit(*args, **kwargs)
return self
def etccv(n_estimators, min_samples_split):
return cross_val_score(AdaBoostClassifier(ETC(
min_samples_split=int(min_samples_split), random_state=2, n_jobs=-1),
algorithm="SAMME",
n_estimators=int(n_estimators)),
train,
train_labels,
'roc_auc',
cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path ="../input/xtrain_v5_full.csv",
test_path="../input/xtest_v5.csv")
# RF
etcBO = BayesianOptimization(etccv, {
'n_estimators': (200, 800),
'min_samples_split': (2, 8)
})
print('-' * 53)
etcBO.maximize()
print('-' * 53)
print('Final Results')
print('ETC: %f' % etcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(ETC(
n_estimators=int(etcBO.res['max']['max_params']['n_estimators']),
import define_prior as dfp
import gvar as gv
import gvar.dataset as gvd
#import importlib as impl ## -- import with variable name
import matplotlib.pyplot as plt
import numpy as np
#import shutil as shil ## -- copy files
import util_funcs as utf
doParallel=True
maxProcesses=6
if df.do_db:
## -- for database input
## - (database file name defined in make_data_db.py)
data,dset = make_data(df.mdp,do_makedata=df.do_makedata,do_db=True)
#models = make_models(data=data,lkey=df.lkey) ## -- make models in loop
else:
## -- for raw correlator file input
data,dset = make_data(df.mdp,do_makedata=df.do_makedata,\
do_db=False,filename="./import-correlators")
## --
mdef = df.define_model
def doProcess(tmin,tmax,data=data,mdef=mdef):
## -- do a single fit ... set up for parallelizing
for key in data:
mdef[key]['tfit'] = range(tmin,tmax)
models = make_models(data=data,lkey=df.lkey,mdef=mdef)
pdict0 = utf.get_prior_dict(df.define_prior,
df.define_prior['nkey'],df.define_prior['okey'],df.num_nst,df.num_ost)
plt.plot([0, 0.5 * fs], [np.sqrt(0.5), np.sqrt(0.5)],
'k--', alpha=0.6, linewidth=1, label=r'$\sqrt{2}/2$')
plt.xlim(0, 0.5*fs)
plt.xlabel('Frequency (Hz)')
plt.ylabel('Gain')
plt.grid(alpha=0.5)
plt.legend(framealpha=1, shadow=True, loc='best')
plt.title("Amplitude response for\nButterworth bandpass filters", fontsize=10)
plt.text(430, 0.07, "lowcut: %4g Hz\nhighcut: %4g Hz" % (lowcut, highcut),
fontsize=8)
plt.tight_layout()
plt.savefig("bandpass_example_response.pdf")
T = 0.03
t, x = make_data(T, fs)
y = butter_bandpass_filt(x, lowcut, highcut, fs, order=12)
plt.figure(figsize=(4.0, 2.8))
plt.plot(t, x, 'k', alpha=0.4, linewidth=1, label='Noisy signal')
plt.plot(t, y, 'C0', label='Filtered signal')
plt.xlabel('Time (seconds)')
plt.grid(alpha=0.5)
plt.axis('tight')
plt.xlim(0, T)
plt.legend(framealpha=1, shadow=True, loc='upper left')
plt.tight_layout()
plt.savefig("bandpass_example_signals.pdf")
def dissemination_RSU():
#일단은 랜덤함수 쓰고
#VDI랑 VTI를 구현해서 RTI를 만들기
#그 값을 계속 변형할 수 있도록 main을 짜기. 알고리즘에 따라
# VDI - VTI - VT - RTI
return RTI
############################### main start ####################################
v1_info = ''
v2_info = ''
v3_info = ''
data = make_data.make_data()
origin_data = data.csv2list()
# origin_packet=''
# print(origin_data)
vehicle_data = data.make_Vdata_list(v1_info, v2_info, v3_info)
# malicious_vehicle(origin_data)
# social_activities(vehicle_data)
#RTI에는 RSU의 모든 데이터가 반영되어 있어야 함. 현재로서는 3대의 차량 정보를 수집했다고 가정
#메인에서 데이터와 데이터 생성 지역 및 시간을 다르게 설정해서 malicious하다고 판단
# RTI: vid, VT, v_position, VT_updateTime
# VDI: vid, VT, data, creationTime, received_data
# VTI: vid, VT, DR, AS, NT, UR, connectivity, VT_updateTime, PVT
import numpy as np
import json
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
from keras import models, layers, optimizers, losses, metrics
from make_data import make_data
# generate baseline data for the first training of the model
winning_boards, winner_history = make_data(model=None,
iterations=1000,
probability=0)
print()
print(
'##############################################################################################################################################'
)
print('Winners of the intial random games')
print(f'Player 1: {winner_history[1]["wins"]/10} %')
print(f'Player 2: {winner_history[-1]["wins"]/10} %')
print(f'Tie Game: {winner_history["tie"]["wins"]/10} %')
print(
'##############################################################################################################################################'
)
print()
all_winning_boards = winning_boards['boards']
all_winning_moves = winning_boards['moves']
for model_iteration in range(5):
X = np.array(all_winning_boards)
np.arange(y_min, y_max, y_mesh_step_size))
values = reg.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
values = values.reshape(xx.shape)
ax.pcolormesh(xx, yy, values, cmap='viridis', vmin=v_min, vmax=v_max)
# Plot the training points, saving the colormap
sctr = ax.scatter(feature_1,
feature_2,
c=y,
cmap='viridis',
edgecolor='black',
lw=0.2)
ax.set_xlim(xx.min(), xx.max())
ax.set_ylim(yy.min(), yy.max())
ax.set_title("Regression predictions (k = {0}, metric = '{1}')".format(
reg.k, reg.distance.__name__))
if __name__ == "__main__":
X, y = make_data(n_features=2, n_pts=300, noise=0.1)
knn = KNNRegressor(k=1, distance=manhattan_distance)
knn.fit(X, y)
y_pred = knn.predict(X)
print(mean_squared_error(y, y_pred))
fig, ax = plt.subplots()
plot_predictions(ax, knn, X, y)
plt.show()
objective="binary:logistic")
# Run Kfolds on the data model to stop over-fitting
X_train, X_valid, y_train, y_valid = train_test_split(train,
train_labels,
test_size=0.1,
random_state=seed)
xgb_model = clf.fit(X_train, y_train, eval_metric="auc", eval_set=[(X_valid, y_valid)], early_stopping_rounds=20)
y_pred = xgb_model.predict_proba(X_valid)[:,1]
return auc(y_valid, y_pred)
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path = "../input/xtrain_v6.csv", test_path="../input/xtest_v6.csv")
xgboostBO = BayesianOptimization(xgboostcv,
{'max_depth': (8, 30),
'learning_rate': (0.8, 0.1),
'n_estimators': (250, 1500),
'gamma': (1., 0.01),
'min_child_weight': (2, 20),
'max_delta_step': (0., 0.3),
'subsample': (0.7, 0.85),
'colsample_bytree': (0.7, 0.85)
})
xgboostBO.maximize(init_points=7, restarts=50, n_iter=30)
print('-' * 53)
