def main(args):
if len(args) not in [3, 4]:
sys.stderr.write("Usage: %s <data file> <visit number> [year]\n" % args[0])
sys.exit(1)
code_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)))
project_root = os.path.join(code_dir, os.path.pardir)
resource_dir = os.path.join(project_root, "resources")
#import pdb; pdb.set_trace()
filename = args[1]
visit = args[2]
output_dir = os.path.join(project_root, "output" + "." + visit)
if len(args) == 4:
calc_year = int(args[3])
else:
now = datetime.datetime.now()
calc_year = now.year
if not os.path.exists(output_dir):
os.mkdir(output_dir)
os.mkdir(os.path.join(output_dir, "scorecard"))
os.mkdir(os.path.join(output_dir, "noscorecard"))
stats_writer = StatsWriter(open("stats.csv", "w"))
context = {}
context["school_map"] = dataloader.load_schooltypes(os.path.join(resource_dir, "school_type.xls"))
context["menu"] = dataloader.load_menu(os.path.join(resource_dir, "menu.xls"))
template_xml = open(os.path.join(resource_dir, svg_template)).read().decode("utf-8")
# load all visit data
all_data = dataloader.load_data(filename, visit, calc_year, context)
all_visit_data = all_data["current_visit"]
for i, school in enumerate(all_visit_data):
print "Processing school: %s" % school.school_number
stats_data = []
stats_data.append(school.school_number)
stats_data.append(school.visit)
stats_data.append(school.stock_score)
stats_data.append(school.meal_delivery_score)
stats_data.append(school.hygiene_score)
stats_data.append(school.staff_score)
stats_data.append(school.total_score)
stats_writer.write_stats(stats_data)
school_xml = template_xml
school_xml = render.render_scorecard(all_data, school, school_xml)
output_path = os.path.join(output_dir, "%s" % ("scorecard" if context["school_map"][school.school_number]["score_card"] == 1 else "noscorecard"))
output_file = "%d.svg" % school.school_number
f = open(os.path.join(output_path, output_file), "w")
f.write(school_xml.encode("utf-8"))
f.close()
def main():
"""
Grabs data from data loader and runs appropriate function
"""
images, labels = load_data(data_dir="./CAFE/")
data = zip(images, labels)
slice_data = [(image, label) for image, label in data
if c2 in label.strip('pgm') or c1 in label.strip('pgm')]
if test_single:
single_test(slice_data)
else:
if test_learning_rates:
learning_rate_test(slice_data)
else:
pca_test(slice_data)
def train(args, yaml_data):
set_random_seed(yaml_data["seed"])
make_results_folder(yaml_data["results_folder"])
model = load_model(yaml_data)
dataloaders = load_data(**yaml_data)
train_dataloader = dataloaders["train"]
val_dataloader = dataloaders.get("val")
num_epochs = yaml_data.get("max_epoch")
results_folder = yaml_data["results_folder"]
losses = run_training(model,
train_data=train_dataloader,
val_data=val_dataloader,
num_epochs=num_epochs,
results_folder=results_folder)
def test_generator_training():
# parameters
file_name = "animals.txt"
genr_hidden_size = 10
disr_hidden_size = 3
num_epochs_d = 20
num_epochs_g = 20
lr = 1
alpha = 0.9
batch_size = 100
# load data
char_list = dataloader.get_char_list(file_name)
X_actual = dataloader.load_data(file_name)
num_examples = X_actual.shape[0]
seq_len = X_actual.shape[1]
# generate
genr_input = np.random.randn(num_examples, len(char_list))
genr = Generator(genr_hidden_size, char_list)
X_generated = genr.generate_tensor(seq_len, num_examples, genr_input)
# train discriminator
disr = Discriminator(len(char_list), disr_hidden_size)
disr.train_RMS(X_actual, X_generated, num_epochs_d, lr, alpha, batch_size)
# evaluate discriminator
accuracy = disr.accuracy(X_actual, X_generated)
print("accuracy: ", accuracy)
# train generator
genr.train_RMS(genr_input,
seq_len,
disr,
num_epochs_g,
1,
lr,
alpha,
batch_size,
print_progress=True)
# evaluate discriminator again
X_generated = genr.generate_tensor(seq_len, num_examples, genr_input)
accuracy = disr.accuracy(X_actual, X_generated)
print("accuracy: ", accuracy)
def check_auc(g_model_path, d_model_path, opt, i):
opt.batch_shuffle = False
opt.drop_last = False
dataloader = load_data(opt)
model = OGNet(opt, dataloader)
model.cuda()
d_results, labels = model.test_patches(g_model_path, d_model_path, i)
d_results = np.concatenate(d_results)
labels = np.concatenate(labels)
fpr1, tpr1, thresholds1 = metrics.roc_curve(labels, d_results, pos_label=1) # (y, score, positive_label)
fnr1 = 1 - tpr1
eer_threshold1 = thresholds1[np.nanargmin(np.absolute((fnr1 - fpr1)))]
eer_threshold1 = eer_threshold1
d_f1 = np.copy(d_results)
d_f1[d_f1 >= eer_threshold1] = 1
d_f1[d_f1 < eer_threshold1] = 0
f1_score = metrics.f1_score(labels, d_f1, pos_label=0)
print("AUC: {0}, F1_score: {1}".format(metrics.auc(fpr1,tpr1), f1_score))
def main():
X_trainfilenames, Y_train, X_devfilenames, Y_dev, X_evalfilenames, Y_eval = dataloader.load_data(
)
X_eval2filenames, Y_eval2 = dataloader.load_other_eval_data()
X_trainGenuineFiles = X_trainfilenames[Y_train == 0][0:N_FILES]
Y_trainGenuine = Y_train[Y_train == 0][0:N_FILES]
X_trainSpoofFiles = X_trainfilenames[Y_train == 1][0:N_FILES]
Y_trainSpoof = Y_train[Y_train == 1][0:N_FILES]
assert (Y_trainSpoof.all() and not Y_trainGenuine.any())
train(X_trainGenuineFiles, X_trainSpoofFiles)
evaluateAndSaveResults(
np.concatenate((X_trainGenuineFiles, X_trainSpoofFiles)),
np.concatenate((Y_trainGenuine, Y_trainSpoof)), X_devfilenames, Y_dev,
X_evalfilenames, Y_eval, X_eval2filenames, Y_eval2)
plotROC(X_evalfilenames, Y_eval)
def __init__(self, preprocessing="", shuffle_train=False, shuffle_test=False):
self.x_train, self.y_train, self.x_test, self.y_test, self.feature_size = load_data(t=preprocessing)
self.train_size = len(self.y_train)
self.test_size = len(self.x_test)
# TODO: reset the prior distribution
self.pi_1 = 0
if shuffle_train:
idx = np.arange(self.train_size)
np.random.shuffle(idx)
self.x_train = [self.x_train[idx[i]] for i in range(self.train_size)]
self.y_train = [self.y_train[idx[i]] for i in range(self.train_size)]
if shuffle_test:
idx = np.arange(self.test_size)
np.random.shuffle(idx)
self.x_test = [self.x_test[idx[i]] for i in range(self.test_size)]
self.y_test = [self.y_test[idx[i]] for i in range(self.test_size)]
def main():
args = parser.parse_args()
config = get_config(args.config)
vgg, model = get_networks(config, load_checkpoint=True)
if config['localization_test']:
test_dataloader, ground_truth = load_localization_data(config)
roc_auc = localization_test(model=model,
vgg=vgg,
test_dataloader=test_dataloader,
ground_truth=ground_truth,
config=config)
else:
_, test_dataloader = load_data(config)
roc_auc = detection_test(model=model,
vgg=vgg,
test_dataloader=test_dataloader,
config=config)
last_checkpoint = config['last_checkpoint']
print("RocAUC after {} epoch:".format(last_checkpoint), roc_auc)
def main():
start_time = time.time()
print("*********Single Layer Neural Network**********")
fifa_dataset, finance_dataset, orbits_dataset = dataloader.load_data()
print("-----------------fifa-----------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(fifa_dataset)
data_set_by_neural_network(train_x, c_train_y,
test_x, c_test_y, "fifa")
print("----------------finance---------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(finance_dataset)
data_set_by_neural_network(train_x, c_train_y,
test_x, c_test_y, "finance")
print("-----------------orbits---------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(orbits_dataset)
data_set_by_neural_network(train_x, c_train_y,
test_x, c_test_y, "orbits")
print("------------------------------------")
elapsed_time = time.time() - start_time
print(elapsed_time, " seconds to complete the task")
def buildImageMap():
# Load and match filenames to images
images, labels = dataloader.load_data()
labeledImages = list(zip(labels, images))
labeledImages.sort(key=lambda tup: tup[0])
imageMap = {}
for label, image in labeledImages:
# Parse semantic from filename
semantic = re.search(r"_([a-z]*)\d*.", label).groups()[0]
# Map semantic to list of images
if semantic not in imageMap:
imageMap[semantic] = []
imageMap[semantic].append((image, semantic))
# Remove neutral emotion
imageMap.pop("n")
return imageMap
def main():
start_time = time.time()
print("*********Linear Regression**********")
fifa_dataset, finance_dataset, orbits_dataset = dataloader.load_data()
print("----------------fifa----------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(
fifa_dataset)
data_set_by_linear_regression(train_x, c_train_y, test_x, c_test_y, "fifa")
print("----------------finance-------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(
finance_dataset)
data_set_by_linear_regression(train_x, c_train_y, test_x, c_test_y,
"finance")
print("-----------------orbits-------------")
train_x, test_x, c_train_y, c_test_y, _, _ = dataloader.dataset_to_table(
orbits_dataset)
data_set_by_linear_regression(train_x, c_train_y, test_x, c_test_y,
"orbits")
elapsed_time = time.time() - start_time
print("------------------------------------")
print(elapsed_time, " seconds to complete the task")
def main():
#data = "hello i am paradox. i am nishan. i am gru"
data = load_data("data/input.txt")
hidden_size = 100
seq_length = 25
learning_rate = 1e-1
int_text, vocab_to_int, int_to_vocab = process_text(data)
vocab_size = len(vocab_to_int)
rnn = RNN(hidden_size, vocab_size, learning_rate)
rnn.train(np.array(int_text), iteration=100, seq_length=seq_length)
sample_idx = rnn.sample(int_text[0], 20)
generated_text = ' '.join(int_to_vocab[idx] for idx in sample_idx)
for key, token in tokens.items():
generated_text = generated_text.replace(' ' + token.lower(), key)
generated_text = generated_text.replace('\n ', '\n')
generated_text = generated_text.replace('( ', '(')
print(generated_text)
def check_auc(g_model_path, d_model_path, i):
opt_auc = parse_opts()
opt_auc.batch_shuffle = False
opt_auc.drop_last = False
opt_auc.data_path = './data/test/'
dataloader = load_data(opt_auc)
model = OGNet(opt_auc, dataloader)
model.cuda()
d_results, labels = model.test_patches(g_model_path, d_model_path, i)
d_results = np.concatenate(d_results)
labels = np.concatenate(labels)
fpr1, tpr1, thresholds1 = metrics.roc_curve(
labels, d_results, pos_label=1) # (y, score, positive_label)
fnr1 = 1 - tpr1
eer_threshold1 = thresholds1[np.nanargmin(np.absolute((fnr1 - fpr1)))]
EER1 = fpr1[np.nanargmin(np.absolute((fnr1 - fpr1)))]
d_f1 = np.copy(d_results)
d_f1[d_f1 >= eer_threshold1] = 1
d_f1[d_f1 < eer_threshold1] = 0
f1_score = metrics.f1_score(labels, d_f1, pos_label=0)
print("AUC: {0}, EER: {1}, EER_thr: {2}, F1_score: {3}".format(
metrics.auc(fpr1, tpr1), EER1, eer_threshold1, f1_score))
def main():
# show_folder_stat(folder + dataset_small)
# show_folder_stat(folder + dataset_large)
def save_data_to_file(dataset_name):
data, results = dataloader.get_all_data(dataset_name)
data, results = dataloader.filter_duplicates(data, results)
dataloader.save_data(dataset_name, data, results)
# save_data_to_file(dataloader.dataset_small)
# save_data_to_file(dataloader.dataset_large)
data, results = dataloader.load_data(dataloader.dataset_small)
# data, results = dataloader.resize_data(data, results, 100)
# data = scale_data(data)
x_train, x_test, y_train, y_test = train_test_split(data,
results,
test_size=0.1,
random_state=42)
x_valid, x_test, y_valid, y_test = train_test_split(x_test,
y_test,
test_size=0.5,
random_state=42)
logreg = LogisticRegression(
solver='lbfgs',
multi_class='multinomial',
# verbose=10,
# max_iter=20
)
logreg.fit(x_train, y_train)
# y_pred = logreg.predict(x_test)
# accuracy = accuracy_score(y_test, y_pred)
# print(f'Test set Accuracy: {accuracy}')
test_score = logreg.score(x_test, y_test)
print(f'Test set Score: {test_score}')
test_score = logreg.score(x_valid, y_valid)
print(f'Validation set Score: {test_score}')
def test_gan():
#parameters
file_name = "animals.txt"
g_hidden_size = 10
d_hidden_size = 10
n_epochs = 1000
g_epochs = 20
d_epochs = 10
g_initial_lr = 1
d_initial_lr = 1
g_multiplier = 0.9
d_multiplier = 0.9
g_batch_size = 100
d_batch_size = 100
# data
char_list = dataloader.get_char_list(file_name)
X_actual = dataloader.load_data(file_name)
seq_len = X_actual.shape[1]
# construct GAN
gan = GAN(g_hidden_size, d_hidden_size, char_list)
# train GAN
gan.train(X_actual,
seq_len,
n_epochs,
g_epochs,
d_epochs,
g_initial_lr,
d_initial_lr,
g_multiplier,
d_multiplier,
g_batch_size,
d_batch_size,
print_progress=True,
num_displayed=3)
def test_discriminator():
# parameters
file_name = "animals.txt"
genr_hidden_size = 10
disr_hidden_size = 11
num_epochs = 20
lr = 1
alpha = 0.9
batch_size = 100
# load data
char_list = dataloader.get_char_list(file_name)
X_actual = dataloader.load_data(file_name)
num_examples = X_actual.shape[0]
seq_len = X_actual.shape[1]
# generate
genr = Generator(genr_hidden_size, char_list)
X_generated = genr.generate_tensor(seq_len, num_examples)
# train discriminator
disr = Discriminator(len(char_list), disr_hidden_size)
disr.train_RMS(X_actual,
X_generated,
num_epochs,
lr,
alpha,
batch_size,
print_progress=True)
# print discriminator output
outp = disr.discriminate(np.concatenate((X_actual, X_generated), axis=0))
print(outp)
# evaluate discriminator
accuracy = disr.accuracy(X_actual, X_generated)
print("accuracy: ", accuracy)
def training(epochs, batch_size):
X_train = load_data()
batch_count = int(X_train.shape[0] / batch_size)
generator= create_generator(learning_rate,beta_1,encoding_dims)
discriminator= create_discriminator(learning_rate,beta_1)
gan = create_gan(discriminator, generator,encoding_dims)
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
seed = np.random.normal(0,1, [25, encoding_dims])
for e in range(1,epochs+1 ):
print("Epoch %d" %e)
for _ in range(batch_count):
noise= np.random.normal(0,1, [batch_size, encoding_dims])
generated_images = generator.predict(noise)
image_batch = X_train[np.random.randint(low=0,high=X_train.shape[0],size=batch_size)]
discriminator.trainable=True
d_loss_real = discriminator.train_on_batch(image_batch, valid)
d_loss_fake = discriminator.train_on_batch(generated_images, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
noise= np.random.normal(0,1, [batch_size, encoding_dims])
discriminator.trainable=False
g_loss = gan.train_on_batch(noise,valid)
print ("%d [D loss: %f] [G loss: %f]" % (e, d_loss, g_loss))
if ipython:
display.clear_output(wait=True)
plot_generated_images(e, generator,seed,outdir)
generator.save('{}/gan_model'.format(outdir))
def train(test_size, file_name, path_name, test_path):
'''
train data for training from csv file housing.csv
Args:
test_size: size of test samples
file_name : namefile of data
path_name : filename where model will be downloaded
'''
logger.warning("Get_data")
try:
data = load_data(file_name)
except FileNotFoundError:
logger.exception("file not found")
logger.warning("Preprocess data")
# preprocessing (utils.py)
logger.warning("Split data to train and test")
data_train, data_test = train_test_split(data,
test_size=test_size,
random_state=5)
logger.warning("test data download")
data_test = pd.DataFrame(data_test)
data_test.to_csv(test_path, index=False)
logger.warning("download test data")
logger.warning("Initialize")
model = Model()
logger.warning("Fit model")
model.trainee(data_train[:, :-1], data_train[:, -1])
model.save(path_name)
logger.warning(f"Model is saved to {path_name}")
def main(config: Config):
transformer = TokensToNumbers()
transformer.load(config.word_index_file)
if config.model == 'ConSSED':
model = ConSSEDModel(config, transformer.word_index)
model.load_weights()
test_sem, test_sen = load_data_for_conssed(config, transformer, 'test')
predictions, predictions_without_others_class_regularizer = model.predict(
test_sem, test_sen)
else:
model = (BiLSTMModel if config.model == 'BiLSTM' else BaselineModel)(
config, transformer.word_index)
model.load_weights()
test = load_data(config, transformer, 'test')
predictions = model.predict(test)
predictions_without_others_class_regularizer = None
create_solution(predictions, config.test_data_path)
if config.metrics_summary:
show_metrics_summary(predictions,
predictions_without_others_class_regularizer,
config)
parser.add_argument("--epoch", type=int, default=50,help="epoch")
parser.add_argument("--lr", type=float, default=1e-3,help="lr")
parser.add_argument("--require_improvement", type=int, default=100,help="require_improvement")
args = parser.parse_args()
########################################################################################################
# directory for tensorboard
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
train_log_dir = 'logs/gradient_tape/' + current_time + '/train'
train_summary_writer = tf.summary.create_file_writer(train_log_dir)
#########################################################################################################
#Preprocessing Data
data_train = load_data()
getter = modifying(data_train)
getter.get_next()
tag2id,n_tags,word2id,n_words = getter.indexing()
text_sequences,label_sequences = getter.padding(args.max_len,word2id,tag2id) # making length of all sentences to be equal
train_dataset = tf.data.Dataset.from_tensor_slices((text_sequences, label_sequences)) # converting to tensorflow dataset
train_dataset = train_dataset.shuffle(len(text_sequences)).batch(args.batch_size, drop_remainder=True)
print("hidden_num:{}, vocab_size:{}, label_size:{}".format(args.hidden_num, len(word2id), len(tag2id)))
#######################################################################################################
model = NerModel(hidden_num = args.hidden_num, vocab_size = len(word2id)+1, label_size= len(tag2id), embedding_size = args.embedding_size)
optimizer = tf.keras.optimizers.Adam(args.lr)
def grid_search(args):
'''
描述：用grid_search找到最优超参数
参数：args
返回：最优的loss_train_list, acc
'''
model_type = args.type
if model_type == 'Linear':
average_choices = [True, False]
reg_choices = [0, 1e-3, 1e-2, 1e-1, 1, 10, 100, 1000]
reg_type_choices = [1, 2]
lr_choices = [1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1]
bs_choices = [1, 16, 32, 64, 128, 256]
else:
average_choices = [True, False]
reg_choices = [0, 1e-2, 1e-1, 1, 10]
reg_type_choices = [2]
lr_choices = [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2]
bs_choices = [1, 16, 32, 64, 128, 256]
best_reg = 0
best_reg_type = 0
best_lr = 0
best_bs = 0
best_acc = 0
best_average = False
best_loss_train_list = []
print("Model Type:", model_type)
for average in average_choices:
for reg in reg_choices:
for reg_type in reg_type_choices:
for lr in lr_choices:
for bs in bs_choices:
data = load_data(args.norm)
if model_type == 'Linear':
the_model = LinearSVM(data,
batch_size=bs,
learning_rate=lr,
epochs=2000,
reg_type=reg_type,
reg_weight=reg,
whether_print=False,
whether_average=average)
loss_train_list, loss_test, acc_test = the_model.run(
)
elif model_type == 'Kernel':
the_model = RbfSVM(data,
batch_size=bs,
learning_rate=lr,
epochs=2000,
reg_type=reg_type,
reg_weight=reg,
whether_print=False,
whether_average=average)
loss_train_list, loss_test, acc_test = the_model.run(
)
the_acc = acc_test
print("Acc", the_acc, "Average:", average, "Reg_type:",
reg_type, "Reg weight:", reg, "lr:", lr, "bs:",
bs)
if (the_acc > best_acc and the_acc < 1):
best_average = average
best_acc = the_acc
best_reg = reg
best_reg_type = reg_type
best_lr = lr
best_bs = bs
best_loss_train_list = loss_train_list
print(
"\nBest accuracy:{:.4f}\nBest average:{}\nBest reg type:{}\nBest reg weight:{}\nBest learning rate:{}\nBest batch size:{}\n"
.format(best_acc, best_average, best_reg_type, best_reg, best_lr,
best_bs))
return best_loss_train_list, best_acc
def main():
do_show_fig = True
do_save_fig = True
CrossValid = True
SoftmaxQ6 = True
StochasticDescent = True #useful if SoftmaxQ6 == True & CompareLrQ5c == False
CompareLrQ5c = False # Ture if for Q5ciii (select different learning rates)
if CompareLrQ5c:
SoftmaxQ6 = False #Disable this flag variable if CompareLrQ5c == True
lr1 = 0.1
lr2 = 3
lr3 = 10
lr = 3 # learning rate lr = 0.02 when using SGD
M = 300 # Total Epoch
Interval = 50 # Interval used for drawing the errorbars
Num_PC = 50 #Number of PCs
if not SoftmaxQ6: #Q5
if CrossValid:
# k-fold dataset
k = 10
# Load data from ./resized/ folder
images, cnt = load_data(data_dir="./aligned/")
Minivan = images.get('Minivan')
Convertible = images.get('Convertible')
minivan = flatten_img(Minivan)
convertible = flatten_img(Convertible)
if not CompareLrQ5c:
cost_train, acc_train, cost_val, acc_val, cost_test, acc_test, final_acc, std_final_acc = CrossRun(k, minivan, convertible, lr, M, Num_PC)
print('The final test accuracy is %f (std = %f)' %(final_acc, std_final_acc))
else:
print('First Learning Rate')
cost_train1, acc_train1, cost_val1, acc_val1, cost_test1, acc_test1, final_acc1, std_final_acc1 = CrossRun(k, minivan, convertible, lr1, M, Num_PC)
print('Second Learning Rate')
cost_train2, acc_train2, cost_val2, acc_val2, cost_test2, acc_test2, final_acc2, std_final_acc2 = CrossRun(k, minivan, convertible, lr2, M, Num_PC)
print('Third Learning Rate')
cost_train3, acc_train3, cost_val3, acc_val3, cost_test3, acc_test3, final_acc3, std_final_acc3 = CrossRun(k, minivan, convertible, lr3, M, Num_PC)
print('The final test accuracy is %f (std = %f) for lr=%f' % (final_acc1, std_final_acc1, lr1))
print('The final test accuracy is %f (std = %f) for lr=%f' % (final_acc2, std_final_acc2, lr2))
print('The final test accuracy is %f (std = %f) for lr=%f' % (final_acc3, std_final_acc3, lr3))
else:
# one run
# Load data from ./resized/ folder
images, cnt = load_data(data_dir="./resized/")
Minivan = images.get('Minivan')
Convertible = images.get('Convertible')
minivan = flatten_img(Minivan)
convertible = flatten_img(Convertible)
cost_train, acc_train, cost_val, acc_val, cost_test, acc_test, final_acc = OneRun(minivan, convertible, lr, M, Num_PC)
print('The final test accuracy is %f' %(final_acc))
else: #Q6
# k-fold dataset
k = 10
# Load data from ./resized/ folder
images, cnt = load_data(data_dir="./aligned/")
Minivan = images.get('Minivan')
Convertible = images.get('Convertible')
Pickup = images.get('Pickup')
Sedan = images.get('Sedan')
minivan = flatten_img(Minivan)
convertible = flatten_img(Convertible)
pickup = flatten_img(Pickup)
sedan = flatten_img(Sedan)
cost_train, acc_train, cost_val, acc_val, cost_test, acc_test, final_acc, std_final_acc, Confusion_Matrix = Softmax(k, minivan, convertible, pickup, sedan, lr, M, Num_PC, StochasticDescent = StochasticDescent)
print('The Confusion Matrix is\n')
print(Confusion_Matrix)
print('The final test accuracy is %f (std = %f)' %(final_acc, std_final_acc))
# Plot: Cost & Accuracy against Epochs
if not CompareLrQ5c:
plotFunc(cost_test, acc_test, SetName = 'TestingSet', do_save_fig = do_save_fig, CrossValid = CrossValid, Softmax = SoftmaxQ6, Epoch = M, Interval = Interval)
plotFunc2(cost_train, cost_val, param='Error', do_save_fig=do_save_fig, CrossValid=CrossValid, Epoch = M, Softmax = SoftmaxQ6, Interval = Interval)
plotFunc2(acc_train, acc_val, param='Accuracy', do_save_fig=do_save_fig, CrossValid=CrossValid, Epoch = M, Softmax = SoftmaxQ6, Interval = Interval)
else:
plotFunc3(cost_train1, cost_train2, cost_train3, lr1, lr2, lr3, SetName = 'TrainSet', do_save_fig = do_save_fig, CrossValid = CrossValid, Softmax = SoftmaxQ6, Epoch = M, Interval = Interval)
if do_show_fig:
plt.show()
print('[%.1f] Finish S2I' % (time.time() - start_time))
if __name__ == '__main__':
global args
# Create folder for saving those images
if not os.path.exists('./images'):
os.makedirs('./images')
# Load data
dict_save = {}
args = parser.parse_args()
if args.data_cate == 1:
ERSP_all, tmp_all, freqs = dataloader.load_data()
elif args.data_cate == 2:
if args.normal_sub:
with open(
'./raw_data/ERSP_from_raw_%d_channel%d_nolog.data' %
(args.subject_ID, args.num_channels), 'rb') as fp:
dict_ERSP = pickle.load(fp)
ERSP_all, tmp_all, freqs, S_all = dict_ERSP['ERSP'], dict_ERSP[
'SLs'], dict_ERSP['freq'], dict_ERSP['Sub_ID']
else:
with open(
'./raw_data/ERSP_from_raw_%d_channel%d.data' %
(args.subject_ID, args.num_channels), 'rb') as fp:
dict_ERSP = pickle.load(fp)
ERSP_all, tmp_all, freqs = dict_ERSP['ERSP'], dict_ERSP[
'SLs'], dict_ERSP['freq']
def main(args):
# Setup tensorboard stuff
writer = SummaryWriter("../tensorboard_data/" + args.model_type + "-" +
args.denoise + str(args.denoise_latent) +
str(args.noise_level) +
str(datetime.datetime.now()))
params_to_tb(writer, args)
## Load Data
size = 2
spectrum, y = load_data(args.channelwise)
if args.benchmark:
spectrum = spectrum[y[:, 0] <= 6000]
y = y[y[:, 0] <= 6000]
spectrum = spectrum[y[:, 0] >= 4000]
y = y[y[:, 0] >= 4000]
size = 4
spectrum, y = interpolate(spectrum, y, number_of_inters=size)
torch.manual_seed(0)
#spectrum = add_noise(spectrum, args.noise_level)
print(spectrum.shape)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if args.model_type == 'bayes' or args.model_type == 'bAttnVGG' or args.model_type == 'bAttn1d':
Bayesian = True
else:
Bayesian = False
X_train, X_test, y_train, y_test = train_test_split(spectrum.data.numpy(),
y.data.numpy(),
random_state=55,
test_size=0.1)
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
random_state=55,
test_size=0.1)
X_train = torch.from_numpy(X_train).float()
y_train = torch.from_numpy(y_train).float()
X_val = torch.from_numpy(X_val).float()
y_val = torch.from_numpy(y_val).float()
X_test = torch.from_numpy(X_test).float()
y_test = torch.from_numpy(y_test).float()
print("Normalizing")
train_means = torch.mean(y_train, dim=0)
train_std = torch.std(y_train, dim=0)
y_train = (y_train - train_means) / train_std
y_val = (y_val - train_means) / train_std
y_test = (y_test - train_means) / train_std
print(train_std)
print(train_means)
print(spectrum.shape)
print(y.shape)
if args.model_type == 'conv1d':
model = conv1D(in_size=spectrum.shape[-1],
out_size=4,
input_channels=spectrum.shape[1],
convolutions=args.convolutions,
kernel_size=args.kernel_size,
hiddenlayer=args.hiddenlayer,
maxpool=args.maxpool,
dropout=args.dropout)
elif args.model_type == 'resnet':
print("resnet")
model = ResidualNetworkD1(in_size=spectrum.shape[-1],
out_size=4,
input_channels=spectrum.shape[2],
convolutions=args.convolutions,
kernel_size=args.kernel_size,
hiddenlayer=args.hiddenlayer,
maxpool=args.maxpool,
dropout=args.dropout)
elif args.model_type == 'conv2d':
print("resnet2d")
model = ResidualNetworkD2(in_size=8 * 4096,
out_size=4,
convolutions=args.convolutions,
kernel_size=args.kernel_size,
hiddenlayer=args.hiddenlayer,
maxpool=args.maxpool,
dropout=args.dropout)
elif args.model_type == 'bayes':
print('Bayesian')
model = BayesianResidualNetworkD1(in_size=spectrum.shape[-1],
out_size=4,
input_channels=spectrum.shape[2],
convolutions=args.convolutions,
kernel_size=args.kernel_size,
hiddenlayer=args.hiddenlayer,
maxpool=args.maxpool,
dropout=args.dropout)
elif args.model_type == 'attention':
print("spatialAttetion")
model = SpatialAttentionNetwork(4)
elif args.model_type == 'AttnVGG':
print("AttnVGG")
model = AttnVGG_after(im_size=4096,
num_classes=4,
attention=True,
normalize_attn=True)
elif args.model_type == 'bAttnVGG':
print("bAttnVGG")
model = bAttnVGG_after(im_size=4096,
num_classes=4,
attention=True,
normalize_attn=args.norm_att)
elif args.model_type == 'bAttn1d':
print("batt1d")
model = bAttnVGG_1d(im_size=4096,
num_classes=4,
attention=True,
normalize_attn=True)
else:
model = conv2D(in_size=8 * 4096,
out_size=4,
convolutions=args.convolutions,
kernel_size=args.kernel_size,
hiddenlayer=args.hiddenlayer,
maxpool=args.maxpool,
dropout=args.dropout)
model.to(device)
if (args.l1):
criterion = nn.L1Loss()
else:
criterion = nn.MSELoss()
if (args.SGD):
optimizer = optim.AdamW(model.parameters(), lr=args.learningrate)
else:
optimizer = optim.Adam(model.parameters(),
lr=args.learningrate,
weight_decay=args.l2)
scheduler = optim.lr_scheduler.MultiStepLR(
optimizer,
milestones=[(args.epochs - args.lr_decay_milestones)],
gamma=args.lr_decay_factor)
if (args.model_type == 'attention'):
lr = 3e-4
optim.Adam([{
'params': model.networks.parameters(),
'lr': lr,
'weight_decay': 10e-5
}, {
'params': model.finals.parameters(),
'lr': lr,
'weight_decay': 10e-5
}, {
'params': model.stn.parameters(),
'lr': lr * 10e-2,
'weight_decay': 10e-5
}])
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[325, 420],
gamma=args.lr_decay_factor)
dataset = torch.utils.data.TensorDataset(X_train, y_train)
dataset_val = torch.utils.data.TensorDataset(X_val, y_val)
dataset_test = torch.utils.data.TensorDataset(X_test, y_test)
BATCH_SIZE = args.batch_size
trainloader = torch.utils.data.DataLoader(dataset,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0,
pin_memory=True)
valloader = torch.utils.data.DataLoader(dataset_val,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0,
pin_memory=True)
testloader = torch.utils.data.DataLoader(dataset_test,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=0,
pin_memory=True)
if (args.denoise != " "):
if (args.denoise == 'VAE1D'):
denoiser = ConvVAE1D(dataset[0][0].squeeze(0).shape,
args.denoise_latent**2)
elif (args.denoise == 'DAE'):
denoiser = ConvDAE(dataset[0][0].shape, args.denoise_latent**2)
elif (args.denoise == 'DAE1d'):
print("DAE1d")
denoiser = DAE1d(dataset[0][0].squeeze(0).shape,
args.denoise_latent**2)
elif (args.denoise == 'VAE2D'):
denoiser = ConvVAE(dataset[0][0].shape, args.denoise_latent**2)
elif (args.denoise == 'AFVAE'):
denoiser = AFVAE(dataset[0][0].shape, args.denoise_latent**2)
denoiser.load_state_dict(
torch.load("../savedmodels/" + args.denoise +
str(args.denoise_latent) + str(args.noise_level) +
".pth",
map_location=torch.device(device)))
denoiser.to(device)
denoiser.eval()
test_spectrum_clean = spectrum[0:15].to(device)
test_spectrum = spectrum[0:15].to(device)
denoised, _ = denoiser.reconstruct(test_spectrum.to(device))
print(
f'MSE_recon: {torch.sum((denoised.cpu()-test_spectrum_clean.cpu())**2)}'
)
print(
f'MSE_noise: {torch.sum((test_spectrum.cpu()-test_spectrum_clean.cpu())**2)}'
)
del test_spectrum_clean
del test_spectrum
del denoised
print("setup Complete")
TB_counter = 0
epochs = args.epochs
start_epoch = 0
if args.restore_checkpoint:
checkpoint = torch.load("../savedmodels/checkpoint" + args.model_type +
"-" + args.denoise + str(args.denoise_latent) +
str(args.noise_level))
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch']
loss = checkpoint['loss']
scheduler.load_state_dict(checkpoint['scheduler'])
for epoch in range(start_epoch, epochs):
train_loss = 0
train_counter = 0
model.train()
for i, (mini_batch_x, mini_batch_y) in enumerate(trainloader):
mini_batch_x = add_noise(mini_batch_x, args.noise_level)
# If denoise run a denoising step
if (args.denoise != " "):
mini_batch_x, _ = denoiser.reconstruct(mini_batch_x.to(device))
optimizer.zero_grad()
#### Forward Pass
y_pred = model(mini_batch_x.to(device))
#### Compute Loss
if Bayesian:
loss = nll_loss(y_pred, mini_batch_y.to(device))
#print(loss.item())
#print(y_pred.mean)
#print(y_pred.stddev)
else:
loss = loss_func(y_pred, mini_batch_y.to(device))
#loss = loss_func(y_pred.squeeze(), mini_batch_y.to(device))
#### Backward pass
loss.backward()
optimizer.step()
train_loss += loss.cpu().data.numpy()
train_counter += 1
scheduler.step()
writer.add_scalar("train_loss",
train_loss / train_counter,
global_step=TB_counter)
TB_counter += 1
if ((epoch) % 10) == 0:
val_loss = 0
val_counter = 0
with torch.set_grad_enabled(False):
model.eval()
for i, (val_batch_x, val_batch_y) in enumerate(valloader):
val_batch_x = add_noise(val_batch_x, args.noise_level)
if (args.denoise != " "):
val_batch_x, _ = denoiser.reconstruct(
val_batch_x.to(device))
if Bayesian:
# just take the mean of the estimates
y_pred_test = model(val_batch_x.to(device)).mean
else:
y_pred_test = model(val_batch_x.to(device))
val_loss += loss_func(y_pred_test.squeeze(),
val_batch_y.to(device))
val_counter += 1
val_loss = (val_loss).cpu().data.numpy() / val_counter
writer.add_scalar("validation_loss",
val_loss,
global_step=TB_counter)
if ((epoch) % 10) == 0:
print('Epoch {}: train_loss: {} Val loss: {}'.format(
epoch, loss, val_loss))
if ((epoch % 25) == 0 and args.model_type == 'bAttnVGG'):
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
'scheduler': scheduler.state_dict()
}, "../savedmodels/checkpoint" + args.model_type + "-" +
args.denoise + str(args.denoise_latent) +
str(args.noise_level))
model.eval()
old_batch = None
old_label = None
with torch.set_grad_enabled(False):
final_val_loss = 0
for i, (val_batch_x, val_batch_y) in enumerate(testloader):
val_batch_x = add_noise(val_batch_x, args.noise_level)
# If denoise run a denoising step
if (args.denoise != " "):
with torch.set_grad_enabled(False):
val_batch_x, _ = denoiser.reconstruct(
val_batch_x.to(device))
if Bayesian:
# just take the mean of the estimates
y_pred_test = model(val_batch_x.to(device)).mean
y_pred_test_std = model(val_batch_x.to(device)).stddev
else:
y_pred_test = model(val_batch_x.to(device))
final_val_loss += loss_func(y_pred_test.squeeze(),
val_batch_y.to(device)).cpu()
y_pred = (y_pred_test.detach().cpu() * train_std) + train_means
y = (val_batch_y.detach().cpu() * train_std) + train_means
if i == 0:
residuals = (y_pred - y).cpu().detach()
if Bayesian:
residuals_stds = y_pred_test_std
else:
residuals = torch.cat([residuals, (y_pred - y).detach()],
dim=0)
if Bayesian:
residuals_stds = torch.cat(
[residuals_stds,
y_pred_test_std.detach()], dim=0)
if i < 3:
with open(
'../residuals/data-' + args.model_type + "-" +
args.denoise + str(args.denoise_latent) +
str(args.noise_level) + '.csv', 'a') as data:
np.savetxt(data,
val_batch_x.view(val_batch_x.shape[0],
-1).cpu().data.numpy(),
delimiter=",")
with open(
'../residuals/labels-' + args.model_type + "-" +
args.denoise + str(args.denoise_latent) +
str(args.noise_level) + '.csv', 'a') as data:
np.savetxt(data,
y.view(y.shape[0], -1).data.numpy(),
delimiter=",")
with open(
'../residuals/residuals-' + args.model_type + "-" +
args.denoise + str(args.denoise_latent) +
str(args.noise_level) + '.csv', 'a') as res:
np.savetxt(res, (y_pred - y).detach(), delimiter=",")
if Bayesian:
with open(
'../residuals/residuals-std-' + args.model_type +
"-" + args.denoise + str(args.denoise_latent) +
str(args.noise_level) + '.csv', 'a') as res:
np.savetxt(
res, (y_pred_test_std.detach().cpu() * train_std),
delimiter=",")
if args.model_type == 'bAttnVGG' or args.model_type == 'AttnVGG' or args.model_type == 'bAttn1d':
model.visual_att(testloader, device, args)
final_val_loss = final_val_loss
final_test_loss = 0
final_counter = 0
with torch.set_grad_enabled(False):
for i, (val_batch_x, val_batch_y) in enumerate(testloader):
val_batch_x = add_noise(val_batch_x, args.noise_level)
if (args.denoise != " "):
val_batch_x, _ = denoiser.reconstruct(val_batch_x.to(device))
if Bayesian:
# just take the mean of the estimates
y_pred_test = model(val_batch_x.to(device)).mean
else:
y_pred_test = model(val_batch_x.to(device))
final_test_loss += loss_func(
y_pred_test.squeeze(),
val_batch_y.to(device)).cpu().data.numpy()
final_counter += 1
final_test_loss = final_test_loss / final_counter
print("final validation loss: {}".format(final_val_loss))
print("final std of residuals from validation set: {}".format(
torch.std(residuals, dim=0).cpu().data.numpy()))
print("final mean squared error: {}".format(
torch.mean(residuals**2, dim=0).cpu().data.numpy()))
print("final RMSE error: {}".format(
torch.sqrt(torch.mean(residuals**2, dim=0)).cpu().data.numpy()))
print("final MAE error: {}".format(
torch.mean(torch.abs(residuals), dim=0).cpu().data.numpy()))
if Bayesian:
print("final unnormed mean std from model: {}".format(
torch.mean(y_pred_test_std.cpu() * train_std,
dim=0).cpu().data.numpy()))
print("STARNET RMSE ")
print("[51.2, 0.081, 0.040] ")
print("STARNET MAE ")
print("[31.2, 0.053, 0.025] ")
print("final test loss: {}".format(final_test_loss))
test_sun(model, train_means, train_std, device)
print("Saving Residuals")
if args.savemodel:
torch.save(model.state_dict(), "../savedmodels/" + args.name)
def main(_run):
args = argparse.Namespace(**_run.config)
args.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
args.best_val_loss = float("inf")
# Load data
train_loader, val_loader = load_data(args)
# Define Network
model = load_model(args)
print(model)
# Define optimizer
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
steps_per_epoch = len(train_loader)
_run.info["steps_per_epoch"] = steps_per_epoch
args.out_dir = "{}/{}/pretrained/".format(
ex_contrastive.observers[0].basedir, _run._id)
os.makedirs(args.out_dir)
args.tensorboard_dir = "{}/{}/tensorboard/".format(
ex_contrastive.observers[0].basedir, _run._id)
os.makedirs(args.tensorboard_dir)
writer = SummaryWriter(log_dir=args.tensorboard_dir)
# Train the network
for epoch in range(1, args.epochs +
1): # loop over the dataset multiple times
model.train()
args.current_epoch = epoch
train_loss_epoch = 0.
running_loss = 0.
for i, data in enumerate(train_loader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data[0].to(args.device), data[1].to(args.device)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
loss = model(inputs, labels)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
train_loss_epoch += loss.item()
k = 20
if i % k == 0: # print every 20 mini-batches
print('[%d, %5d] loss: %.3f' %
(epoch, i + 1, running_loss / k))
step = epoch * steps_per_epoch + i + 1
_run.log_scalar("training.loss.step", running_loss / k, step)
writer.add_scalar("Loss Steps/train", running_loss / k, step)
running_loss = 0.
# compute validation loss
val_loss_epoch = compute_val(model, val_loader, args)
_run.log_scalar("val.loss.epoch", val_loss_epoch)
_run.log_scalar("train.loss.epoch", train_loss_epoch / steps_per_epoch)
writer.add_scalar("Learning_rate", args.lr, epoch)
writer.add_scalars(
"Loss vs Epoch", {
"train_loss": train_loss_epoch / steps_per_epoch,
"val_loss": val_loss_epoch
}, epoch)
print("epoch: {}".format(epoch))
print("val.loss.epoch", val_loss_epoch)
print("train.loss.epoch", train_loss_epoch / steps_per_epoch)
print('Finished Training')
def main(index_exp, index_split):
faulthandler.enable()
torch.cuda.empty_cache()
best_error = 100
lr_step = [40, 70, 120]
multiframe = ['convlstm', 'convfc']
dirName = '%s_data%d_%s_%s_%s'%(args.model_name, args.data_cate, args.augmentation, args.loss_type, args.file_name)
fileName = '%s_split%d_exp%d'%(dirName, index_split, index_exp)
# Create folder for results of this model
if not os.path.exists('./results/%s'%(dirName)):
os.makedirs('./results/%s'%(dirName))
# ------------- Wrap up dataloader -----------------
if args.input_type == 'signal':
X, Y_reg, C = raw_dataloader.read_data([1,2,3], list(range(11)), channel_limit=21, rm_baseline=True)
num_channel = X.shape[1]
num_feature = X.shape[2] # Number of time sample
# Remove trials
X, Y_reg = preprocessing.remove_trials(X, Y_reg, threshold=60)
# Split data for cross validation
if args.num_fold == 1:
train_data, test_data, train_target, test_target = train_test_split(X, Y_reg, test_size=0.1, random_state=23)
# Random state 15: training error becomes lower, testing error becomes higher
else:
kf = KFold(n_splits=args.num_fold, shuffle=True, random_state=23)
for i, (train_index, test_index) in enumerate(kf.split(X)):
if i == index_exp:
train_data, train_target = X[train_index, :], Y_reg[train_index]
test_data, test_target = X[test_index, :], Y_reg[test_index]
# Split data for ensemble methods
if not args.ensemble:
if args.num_split > 1:
data_list, target_list = preprocessing.stratified_split(train_data, train_target, n_split=args.num_split, mode=args.split_mode)
train_data, train_target = data_list[index_split], target_list[index_split]
'''
kf = KFold(n_splits=args.num_split, shuffle=True, random_state=32)
for i, (other_index, split_index) in enumerate(kf.split(train_data)):
if i == index_split:
train_data, train_target = train_data[split_index, :], train_target[split_index]
'''
# Normalize the data
if args.normalize:
train_data, test_data = preprocessing.normalize(train_data, test_data)
# Data augmentation
if args.augmentation == 'overlapping':
train_data, train_target = data_augmentation.aug(train_data, train_target, args.augmentation,
(256, 64, 128))
test_data, test_target = data_augmentation.aug(test_data, test_target, args.augmentation,
(256, 64, 128))
elif args.augmentation == 'add_noise':
train_data, train_target = data_augmentation.aug(train_data, train_target, args.augmentation,
(30, 1))
elif args.augmentation == 'add_noise_minority':
train_data, train_target = data_augmentation.aug(train_data, train_target, args.augmentation,
(30, 1))
elif args.augmentation == 'SMOTER':
train_data, train_target = data_augmentation.aug(train_data, train_target, args.augmentation)
# scale data
if args.scale_data:
train_data, test_data = train_data.reshape((train_data.shape[0],-1)), test_data.reshape((test_data.shape[0],-1))
train_data, test_data = preprocessing.scale(train_data, test_data)
train_data = train_data.reshape((train_data.shape[0],num_channel, -1))
test_data = test_data.reshape((test_data.shape[0],num_channel, -1))
if args.model_name in ['eegnet', 'eegnet_trans_signal']:
# (sample, channel, time) -> (sample, channel_NN, channel_EEG, time)
[train_data, test_data] = [X.reshape((X.shape[0], 1, num_channel, num_feature)) \
for X in [train_data, test_data]]
(train_dataTS, train_targetTS, test_dataTS, test_targetTS) = map(
torch.from_numpy, (train_data, train_target, test_data, test_target))
[train_dataset,test_dataset] = map(\
Data.TensorDataset, [train_dataTS.float(),test_dataTS.float()], [train_targetTS.float(),test_targetTS.float()])
if not args.str_sampling:
train_loader = Data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
test_loader = Data.DataLoader(test_dataset, batch_size=args.batch_size)
model_param = [train_data.shape]
elif args.input_type == 'power':
if args.data_cate == 1:
ERSP_all, tmp_all, freqs = dataloader.load_data()
elif args.data_cate == 2:
data_file = './raw_data/ERSP_from_raw_%d_channel21.data'%(args.index_sub)
with open(data_file, 'rb') as fp:
dict_ERSP = pickle.load(fp)
ERSP_all, tmp_all = dict_ERSP['ERSP'], dict_ERSP['SLs']
num_channel = ERSP_all.shape[1]
num_freq = ERSP_all.shape[2]
# Remove trials
ERSP_all, tmp_all = preprocessing.remove_trials(ERSP_all, tmp_all, threshold=60)
# Split data for cross validation
if args.num_fold == 1:
train_data, test_data, train_target, test_target = train_test_split(ERSP_all, tmp_all[:,2], test_size=0.1, random_state=23)
else:
kf = KFold(n_splits=args.num_fold, shuffle=True, random_state=23)
for i, (train_index, test_index) in enumerate(kf.split(ERSP_all)):
if i == index_exp:
train_data, test_data = ERSP_all[train_index, :], ERSP_all[test_index, :]
if args.data_cate == 2:
train_target, test_target = tmp_all[train_index], tmp_all[test_index]
else:
train_target, test_target = tmp_all[train_index, 2], tmp_all[test_index, 2]
if args.add_CE:
assert args.data_cate == 2
with open('./raw_data/CE_sub%d'%(args.index_sub), 'rb') as fp:
CE = pickle.load(fp)
CE_train, CE_test = CE[train_index,:], CE[test_index,:]
# PCA for CE
pca = PCA(n_components=10)
pca.fit(CE_train)
CE_train, CE_test = pca.transform(CE_train), pca.transform(CE_test)
# Split data for ensemble methods
if not args.ensemble:
if args.num_split > 1:
data_list, target_list = preprocessing.stratified_split(train_data, train_target, n_split=args.num_split, mode=args.split_mode)
train_data, train_target = data_list[index_split], target_list[index_split]
'''
kf = KFold(n_splits=args.num_split, shuffle=True, random_state=32)
for i, (other_index, split_index) in enumerate(kf.split(np.arange(len(train_data)))):
if i == index_split:
train_data, train_target = train_data[split_index, :], train_target[split_index]
'''
# Concatenate train and test for standardizinsg
data = np.concatenate((train_data, test_data), axis=0)
target = np.concatenate((train_target, test_target))
# Standardize data
num_train = len(train_data)
data, target = preprocessing.standardize(data, target, train_indices = np.arange(num_train), threshold=0.0)
data = data.reshape((data.shape[0], -1))
# Scale target between 0 and 1
if args.post_scale:
print('Scale the target between 0-1')
target = target/60
# Split data
train_data, test_data = data[:num_train, :], data[num_train:, :]
train_target, test_target = target[:num_train], target[num_train:]
# Data augmentation
if args.augmentation == 'SMOTER':
train_data, train_target = data_augmentation.aug(train_data, train_target, args.augmentation)
# center data
if args.center_flag:
train_data, test_data = preprocessing.center(train_data, test_data)
# scale data
if args.scale_data:
train_data, test_data = preprocessing.scale(train_data, test_data)
# Add conditional entropy
if args.add_CE:
train_data = np.concatenate((train_data, CE_train), axis=1)
test_data = np.concatenate((test_data, CE_train), axis=1)
if args.model_name == 'eegnet_trans_power':
# (sample, channel, freq) -> (sample, channel_NN, channel_EEG, freq)
[train_data, test_data] = [X.reshape((X.shape[0], 1, num_channel, num_freq)) \
for X in [train_data, test_data]]
(train_dataTS, train_targetTS, test_dataTS, test_targetTS) = map(
torch.from_numpy, (train_data, train_target, test_data, test_target))
[train_dataset,test_dataset] = map(\
Data.TensorDataset, [train_dataTS.float(),test_dataTS.float()], [train_targetTS.float(),test_targetTS.float()])
if not args.str_sampling:
train_loader = Data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
test_loader = Data.DataLoader(test_dataset, batch_size=args.batch_size)
model_param = [train_data.shape]
elif args.input_type == 'image':
if args.ensemble:
input_model_name = args.pre_model_name
else:
input_model_name = args.model_name
assert (input_model_name in multiframe) == (args.num_time>1)
# Let input size be 224x224 if the model is vgg16
if input_model_name in ['vgg16', 'resnet50']:
input_size = 224
else:
input_size = 64
# Load Data
data_transforms = {
'train': transforms.Compose([
ndl.Rescale(input_size, args.num_time),
ndl.ToTensor(args.num_time)]),
'test': transforms.Compose([
ndl.Rescale(input_size, args.num_time),
ndl.ToTensor(args.num_time)])
}
print("Initializing Datasets and Dataloaders...")
# Create training and testing datasets
# image_datasets = {x: ndl.TopoplotLoader(args.image_folder, x, args.num_time, data_transforms[x],
# scale=args.scale_image, index_exp=index_exp, index_split=index_split) for x in ['train', 'test']}
[train_dataset,test_dataset] = [ndl.TopoplotLoader(args.image_folder, x, args.num_time, data_transforms[x],
scale=args.scale_image, index_exp=index_exp, index_split=index_split) for x in ['train', 'test']]
# Create training and testing dataloaders
# if not args.str_sampling:
# train_loader = Data.DataLoader(image_datasets['train'], batch_size=args.batch_size, shuffle=True, num_workers=4)
# test_loader = Data.DataLoader(image_datasets['test'], batch_size=args.batch_size, shuffle=False, num_workers=4)
if not args.str_sampling:
train_loader = Data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4)
test_loader = Data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4)
model_param = [input_size]
elif args.input_type == 'EEGLearn_img':
# Load data
with open('./EEGLearn_imgs/data1.data', 'rb') as fp:
dict_data = pickle.load(fp)
data, target = dict_data['data'], dict_data['target']
input_size = data.shape[2]
# Split data for cross validation
if args.num_fold == 1:
train_data, test_data, train_target, test_target = train_test_split(data, target, test_size=0.1, random_state=23)
# Random state 15: training error becomes lower, testing error becomes higher
else:
kf = KFold(n_splits=args.num_fold, shuffle=True, random_state=23)
for i, (train_index, test_index) in enumerate(kf.split(data)):
if i == index_exp:
train_data, train_target = data[train_index, :], target[train_index]
test_data, test_target = data[test_index, :], target[test_index]
(train_dataTS, train_targetTS, test_dataTS, test_targetTS) = map(
torch.from_numpy, (train_data, train_target, test_data, test_target))
[train_dataset,test_dataset] = map(\
Data.TensorDataset, [train_dataTS.float(),test_dataTS.float()], [train_targetTS.float(),test_targetTS.float()])
if not args.str_sampling:
train_loader = Data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
test_loader = Data.DataLoader(test_dataset, batch_size=args.batch_size)
# ------------ Create model ---------------
if args.input_type in ['image','EEGLearn_img']:
model_param = [input_size]
else:
model_param = [train_data.shape]
if not args.ensemble:
model = read_model(args.model_name, model_param)
else:
pre_models = []
for i in range(args.num_split):
pre_model = read_model(args.pre_model_name, model_param)
pre_model.load_state_dict( torch.load('%s/last_model_exp%d_split%d.pt'%(args.ensemble, index_exp, i)) )
set_parameter_requires_grad(pre_model, True)
pre_models.append(pre_model)
model = models.__dict__[args.model_name](pre_models)
print('Use model %s'%(args.model_name))
# Run on GPU
model = model.to(device=device)
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
# define loss function (criterion) and optimizer
if args.loss_type == 'L2':
criterion = nn.MSELoss().to(device=device)
elif args.loss_type == 'L1':
criterion = nn.L1Loss().to(device=device)
elif args.loss_type == 'L4':
criterion = L4Loss
elif args.loss_type == 'MyLoss':
criterion = MyLoss
print('Use %s loss'%(args.loss_type))
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr_rate,momentum=0.9)
#optimizer = torch.optim.Adam(model.parameters(), lr=args.lr_rate)
# Record loss and accuracy of each epoch
dict_error = {'train_std': list(range(args.num_epoch)), 'test_std': list(range(args.num_epoch)),
'train_mape': list(range(args.num_epoch)), 'test_mape': list(range(args.num_epoch))}
# optionally evaluate the trained model
if args.evaluate:
if args.resume:
if os.path.isfile(args.resume):
model.load_state_dict(torch.load(args.resume))
_, target, pred, _, _ = validate(test_loader, model, criterion)
plot_scatter(target, pred, dirName, fileName)
return 0
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_error = checkpoint['best_error']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
dict_error['train_std'][:args.start_epoch] = checkpoint['dict_error']['train_std']
dict_error['test_std'][:args.start_epoch] = checkpoint['dict_error']['test_std']
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
# ------------- Train model ------------------
for epoch in range(args.start_epoch, args.num_epoch):
# Create dataloader if using stratified sampler
if args.str_sampling:
sampler = SubsetRandomSampler(get_indices_RSS(train_target, int(0.5*len(train_target))))
train_loader = Data.DataLoader(train_dataset, batch_size=args.batch_size, \
sampler=sampler, num_workers=4)
# Learning rate decay
if epoch in lr_step:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.1
# train for one epoch
_, dict_error['train_std'][epoch], dict_error['train_mape'][epoch] = \
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
_, _, _, std_error, dict_error['test_mape'][epoch] = validate(test_loader, model, criterion)
dict_error['test_std'][epoch] = std_error
# remember best standard error and save checkpoint
is_best = std_error < best_error
best_error = min(std_error, best_error)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_error': best_error,
'optimizer': optimizer.state_dict(),
'dict_error': dict_error
}, is_best)
# Save best model
if is_best:
torch.save(model.state_dict(), './results/%s/best_model_exp%d_split%d.pt'%(dirName, index_exp, index_split))
if epoch == args.num_epoch-1:
torch.save(model.state_dict(), './results/%s/last_model_exp%d_split%d.pt'%(dirName, index_exp, index_split))
# Plot error curve
plot_error(dict_error, dirName, fileName)
# Plot scatter plots
_, target, pred, _, _ = validate(test_loader, model, criterion)
plot_scatter(target, pred, dirName, fileName)
dict_error['target'], dict_error['pred'] = target, pred
# Plot histogram
import matplotlib.pyplot as plt
plt.hist(target, label = 'True')
plt.hist(pred, label = 'Pred')
plt.legend(loc='upper right')
plt.savefig('./results/hist.png')
# Save error over epochs
with open('./results/%s/%s.data'%(dirName, fileName), 'wb') as fp:
pickle.dump(dict_error, fp)
import warnings
warnings.filterwarnings("ignore")
import os
import numpy as np
import pandas as pd
from collections import Counter
from sklearn.datasets import load_svmlight_file
from LambdaRankNN import RankNetNN
from dataloader import load_data
#dst_path = '/bigtemp/ms5sw/MQ2008-agg/Fold1'
dst_path = '/bigtemp/ms5sw/Fold1/'
# Read dataset
train_X, train_y, train_queries = load_data(dst_path + "train.txt")
test_X, test_y, test_queries = load_data(dst_path + "test.txt")
print("Loaded data")
#print(train_X[:5], train_y[:5], train_queries[:5])
print(train_X.shape, train_y.shape)
#train_y = train_y.astype(int)
#test_y = test_y.astype(int)
#train_X, train_y, train_queries = train_X[:100000,:], train_y[:100000], train_queries[:100000]
#test_X, test_y, test_queries = test_X[:100000,:], test_y[:100000], test_queries[:100000]
# Train RankNetNN model
ranker = RankNetNN(input_size=train_X.shape[1],
hidden_layer_sizes=(
16,
]]
np.savetxt(g,
neighbors_labels,
delimiter=',',
fmt="%s")
np.savetxt(f, [img_id[b]], newline=',', fmt="%s")
neighbors = [[data_index[ind] for ind in indices]]
np.savetxt(f, neighbors, delimiter=' ', fmt="%s")
# if i >= 10:
# break
if __name__ == '__main__':
model_path = '/home/kylecshan/saved/06_01_2019/classification/densenet169/model1_epoch9.pth'
save_path = '/home/kylecshan/saved/06_01_2019/classification/densenet169/densenetclass/densenetclass'
model_name = "densenet_class"
output_dim = 2000
batch_size = 256
dataloaders_dict = load_data(batch_size=batch_size)
model = load_model(model_name, model_path)
# save_index(model, output_dim, dataloaders_dict, save_path, batch_size = batch_size)
retrieve(model,
output_dim,
dataloaders_dict,
save_path,
batch_size=batch_size)
hp.Float('dropout', min_value=0.1, max_value=0.5, step=0.1)),
layers.GlobalAveragePooling1D(),
layers.Dropout(
hp.Float('dropout', min_value=0.1, max_value=0.5, step=0.1)),
layers.Dense(4, activation=activation),
])
model.compile(loss=losses.SparseCategoricalCrossentropy(from_logits=True),
optimizer='adam',
metrics=['accuracy'])
return model
if __name__ == "__main__":
train_ds, val_ds, test_ds = load_data()
AUTOTUNE = tf.data.experimental.AUTOTUNE
train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)
val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)
test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)
tuner = kt.Hyperband(build_model,
objective='val_accuracy',
max_epochs=10,
factor=3,
directory='my_dir',
project_name='second')
tuner.search(train_ds, epochs=50, validation_data=val_ds)
best_hps = tuner.get_best_hyperparameters(num_trials=2)[0]
model = tuner.hypermodel.build(best_hps)
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("--model", type=str, default="cox")
parser.add_argument("--dataset", type=str, default="combined")
parser.add_argument("--validate-on", type=str, default="")
parser.add_argument("--bootstrap-samples", type=int, default=250)
parser.add_argument("--cens-time", type=float, default=365.25 * 3)
args = parser.parse_args()
print("=" * 79)
print(f"== Running for: {args.dataset.upper()}")
if args.dataset == "combined":
dataset = combine_datasets(load_data("sprint"), load_data("accord"))
else:
dataset = load_data(args.dataset)
stats = defaultdict(list)
for _ in tqdm(range(args.bootstrap_samples)):
idxs = bootstrap_dataset(dataset)
bootstrap = {
"X": dataset["X"][idxs],
"y": dataset["y"][idxs],
"w": dataset["w"][idxs],
"t": dataset["t"][idxs]
}
for k, v in run_for_optimism(dataset, bootstrap, args).items():
stats[k].append(v)
def train_lstm(self,
saveto, # The best model will be saved there
dataset,
#----------------------------------------------------------------------
#algorithmic hyperparameters
encoder='lstm', # TODO: can be removed must be lstm.
l2_reg_U=0., # Weight decay for the classifier applied to the U weights.
lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop)
optimizer="adadelta", # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate).
batch_size=16, # The batch size during training.
wemb_init='word2vec',
#----------------------------------------------------------------------
#parameters related to convergence, saving, and similar
max_epochs=5000, # The maximum number of epoch to run
patience=10, # Number of epoch to wait before early stop if no progress
dispFreq=10, # Display to stdout the training progress every N updates
n_words=10000, # Vocabulary size
validFreq=370, # Compute the validation error after this number of update.
saveFreq=1110, # Save the parameters after every saveFreq updates
valid_batch_size=64, # The batch size used for validation/test set.
#----------------------------------------------------------------------
# Parameter for extra option (whatever that means)
noise_std=0.,
use_dropout=True, # if False slightly faster, but worst test error
# This frequently need a bigger model.
reload_model=None, # Path to a saved model we want to start from.
return_after_reloading=False, # Path to a saved model we want to start from.
test_size=-1, # If >0, we keep only this number of test example.
):
optimizer = OPTIMIZERS[optimizer]
# Model options
self.model_options = locals().copy()
if reload_model:
self.faulty_load_params(reload_model)
# self.init_tparams()
_, self.wdim = self.params['Wemb'].shape
self.hdim, ydim = self.params['U'].shape
self.model_options['ydim'] = ydim
print _, self.wdim, self.hdim, ydim
self.model_options['hdim'] = self.hdim
self.model_options['wdim'] = self.wdim
self.model_options['grad_clip_thresh'] = self.grad_clip_thresh
print "model options", self.model_options
# load_data, prepare_data = get_dataset(dataset)
print 'Loading data'
#each of the below is a tuple of
# (list of sentences, where each is a list fo word indices,
# list of integer labels)
if not reload_model:
train, valid, test = load_data(n_words=n_words, valid_portion=0.05,
maxlen=self.maxlen, path=dataset)
if test_size > 0:
# The test set is sorted by size, but we want to keep random
# size example. So we must select a random selection of the
# examples.
idx = numpy.arange(len(test[0]))
numpy.random.shuffle(idx)
idx = idx[:test_size]
test = ([test[0][n] for n in idx], [test[1][n] for n in idx])
ydim = numpy.max(train[1]) + 1
self.model_options['ydim'] = ydim
print 'Building model'
if not reload_model:
# initialize the word embedding matrix and the parameters of the model (U and b) randomly
# self.params is a dict mapping name (string) -> numpy ndarray
self.init_params(self.model_options)
# This creates Theano Shared Variable from the parameters.
# Dict name (string) -> Theano Tensor Shared Variable
# self.params and self.tparams have different copy of the weights.
self.init_tparams()
# use_noise is for dropout
(use_noise, x, mask, y) =\
self.build_model(self.model_options,)
# f_pred_prob, self.f_pred, cost)
if l2_reg_U > 0.:
l2_reg_U = theano.shared(numpy_floatX(l2_reg_U), name='l2_reg_U')
weight_decay = 0.
weight_decay += (self.tparams['U'] ** 2).sum()
weight_decay *= l2_reg_U
self.cost += weight_decay
f_cost = theano.function([x, mask, y], self.cost, name='f_cost')
grads = tensor.grad(self.cost, wrt=self.tparams.values())
f_grad = theano.function([x, mask, y], grads, name='f_grad')
lr = tensor.scalar(name='lr')
f_grad_shared, f_update = optimizer(lr, self.tparams, grads,
x, mask, y, self.cost)
if self.debug:
util.colorprint("Following is the graph of the shared gradient function (f_grad_shared):", "blue")
theano.printing.debugprint(f_grad_shared.maker.fgraph.outputs[0])
if return_after_reloading:
self.model_has_been_trained = True
return
print 'Optimization'
kf_valid = self.get_minibatches_idx(len(valid[0]), valid_batch_size)
kf_test = self.get_minibatches_idx(len(test[0]), valid_batch_size)
print "%d train examples" % len(train[0])
print "%d valid examples" % len(valid[0])
print "%d test examples" % len(test[0])
history_errs = []
best_p = None
bad_count = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
uidx = 0 # the number of update done
estop = False # early stop
start_time = time.time()
try:
for epoch in xrange(max_epochs):
sys.stdout.flush()
n_samples = 0
# Get new shuffled index for the training set.
minibatches = self.get_minibatches_idx(len(train[0]), batch_size, shuffle=True)
for _, train_index_list in minibatches:
uidx += 1
use_noise.set_value(1.)
# Select the random examples for this minibatch
y = [train[1][t] for t in train_index_list]
x = [train[0][t]for t in train_index_list]
# Get the data in numpy.ndarray format
# This swap the axis!
# Return something of shape (minibatch maxlen, n samples)
x, mask, y = prepare_data(x, y)
n_samples += x.shape[1]
cur_cost_val = f_grad_shared(x, mask, y)
f_update(lrate)
if numpy.isnan(cur_cost_val) or numpy.isinf(cur_cost_val):
print 'NaN detected'
return 1., 1., 1.
if numpy.mod(uidx, dispFreq) == 0:
print 'Epoch ', epoch, 'Update ', uidx, 'Cost ', cur_cost_val
if saveto and numpy.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
self.params = best_p
else:
self.params = self.unzip(self.tparams)
numpy.savez(saveto, history_errs=history_errs, **self.params)
pkl.dump(self.model_options, open('%s.pkl' % saveto, 'wb'), -1)
print 'Done'
if numpy.mod(uidx, validFreq) == 0:
use_noise.set_value(0.)
train_err = self.pred_error(self.f_pred, prepare_data, train, minibatches)
valid_err = self.pred_error(self.f_pred, prepare_data, valid,
kf_valid)
test_err = self.pred_error(self.f_pred, prepare_data, test, kf_test)
history_errs.append([valid_err, test_err])
if (uidx == 0 or
valid_err <= numpy.array(history_errs)[:,
0].min()):
best_p = self.unzip(self.tparams)
bad_counter = 0
print ('Train ', train_err, 'Valid ', valid_err,
'Test ', test_err)
if (len(history_errs) > patience and
valid_err >= numpy.array(history_errs)[:-patience,
0].min()):
bad_counter += 1
if bad_counter > patience:
print 'Early Stop!'
estop = True
break
print 'Seen %d samples' % n_samples
if estop:
break
except KeyboardInterrupt:
print "Training interrupted"
end_time = time.time()
if best_p is not None:
self.zipp(best_p, self.tparams)
else:
best_p = self.unzip(self.tparams)
use_noise.set_value(0.)
kf_train_sorted = self.get_minibatches_idx(len(train[0]), batch_size)
train_err = self.pred_error(self.f_pred, prepare_data, train, kf_train_sorted)
valid_err = self.pred_error(self.f_pred, prepare_data, valid, kf_valid)
test_err = self.pred_error(self.f_pred, prepare_data, test, kf_test)
print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err
if saveto:
numpy.savez(saveto, train_err=train_err,
valid_err=valid_err, test_err=test_err,
history_errs=history_errs, **best_p)
print 'The code run for %d epochs, with %f sec/epochs' % (
(epoch + 1), (end_time - start_time) / (1. * (epoch + 1)))
print >> sys.stderr, ('Training took %.1fs' %
(end_time - start_time))
self.model_has_been_trained = True
return train_err, valid_err, test_err
import tensorflow.compat.v1 as tf
import argparse
import numpy as np
import os
from tensorflow.examples.tutorials.mnist import input_data
import argparse
from dataloader import load_data
change_rate = [0.005, 0.05, 0.5, 5, 50] # 参数乘以4
parser = argparse.ArgumentParser(description='')
parser.add_argument('--data', default='mnist')
args = parser.parse_args()
X_train, X_test, y_train, y_test = load_data(args.data)
if args.data == 'mnist':
ori_ckpt_path = 'model/mnist_mlp.ckpt'
new_checkpoint_path = 'model/changed/mnist_mlp_chg.ckpt'
else:
if args.data == "cifar10":
ori_ckpt_path = 'model/cifar_cnn.ckpt'
new_checkpoint_path = 'model/changed/cifar_cnn_chg.ckpt'
def change(change_rate):
with tf.Session(graph=tf.Graph()) as sess: #添加了graph=tf.Graph()可以加载多个模型
'''
new_var_list=[] #新建一个空列表存储更新后的Variable变量
for var_name, _ in tf.train.list_variables(args.checkpoint_path): #得到checkpoint文件中所有的参数（名字，形状）元组
var = tf.train.load_variable(args.checkpoint_path, var_name) #得到上述参数的值
def create_and_save_adversarial_examples(self,
saved_model_fpath,
n_examples=100,
dataset="data/imdb.pkl",
saveto = "output/adversarial_examples.npz",
):
"""
recreates the model from saved parameters, then finds adversarial examples.
right now, not especially modular :(
Allen's note: n_examples is not used
:param string model_fname: the name of the file where the model has been stored.
"""
# below: assert that the training has been done
assert self.model_has_been_trained
# we want to have trained nonadversarially in order to have
# examples that are demonstrative of adversarialness
assert not self.adversarial
(_, x_sym, mask_sym, y_sym) =\
self.build_model(self.model_options,)
grad_wrt_emb = tensor.grad(self.cost, wrt=self.emb)[0]
anti_example = tensor.sgn(grad_wrt_emb)
adv_example = self.emb + self.adv_epsilon*anti_example
f_adv_example = theano.function([x_sym, mask_sym, y_sym], adv_example, name='f_adv_example')
f_identity = theano.function([x_sym], self.emb, name='f_identity')
# 1. get the data
print 'Loading data'
#TODO: remove magic 10000!!!
train, valid, test = load_data(n_words=10000, valid_portion=0.05,
maxlen=self.maxlen, path=dataset)
corpus = valid
# make a datastructure in which to store them
print len(corpus[1])
sentences_and_adversaries = {
'original_sentences': None,
'adversarial_sentences': None,
'saved_model_fpath' : saved_model_fpath,
#metadata
'n_sentences': len(corpus[1]),
'adversarial_parameters': {
'alpha':self.adv_alpha,
'epsilon':self.adv_epsilon,
},
}
x_itf, mask_itf, y_itf = prepare_data(corpus[0], corpus[1])
# print f_adv_example(x_itf, mask_itf, y_itf)
# print f_adv_example(x_itf, mask_itf, y_itf).shape
sentences_and_adversaries['adversarial_sentences'] = f_adv_example(x_itf, mask_itf, y_itf)
sentences_and_adversaries['original_sentences'] = f_identity(x_itf)
numpy.savez(saveto, sentences_and_adversaries)#, open(saveto, 'wb'))
