def main(config):
# Create test dataloader
test_ds_cfg = config['test_data_loader']['args']
test_dataset = MNISTDataset(test_ds_cfg['root'],
test_ds_cfg['csv_file'],
False,
transforms=transforms)
test_loader = DataLoader(test_dataset,
test_ds_cfg['batch_size'],
test_ds_cfg['shuffle'],
num_workers=test_ds_cfg['num_workers'])
# Create classification model
model = CNNModel()
# Create criterion (loss function)
criterion = getattr(loss_md, config['loss'])
# Create metrics for evaluation
metrics = [getattr(loss_md, x) for x in config['metrics']]
# Create optimizer
optimizer = getattr(otpm, config['optimizer']['name'])(
model.parameters(), **config['optimizer']['args'])
# Create learning rate scheduler
lr_scheduler = getattr(otpm.lr_scheduler, config['lr_scheduler']['name'])\
(optimizer, **config['lr_scheduler']['args'])
# Create train procedure: classification trainer
csf_trainer = ClassificationTrainer(config, model, criterion, metrics,
optimizer, lr_scheduler)
csf_trainer.test(test_loader)
def __init__(self, input_path, output_path):
self.input_path = input_path
self.output_path = output_path
set_seed()
if not os.listdir(input_path):
raise NameError("Input directory is empty")
if not os.path.isdir(output_path):
os.mkdir(output_path)
# For speed optimization, all algorithms are defined here
self.backSub = cv.createBackgroundSubtractorMOG2()
self.CNN = CNNModel()
def train():
pos_data_path = '../dataset/weibo60000/pos60000_utf8.txt_updated'
pos_x, pos_y = read_pos_data(pos_data_path)
print(len(pos_x))
print(len(pos_y))
# print(pos_y)
neg_data_path = '../dataset/weibo60000/neg60000_utf8.txt_updated'
neg_x, neg_y = read_neg_data(neg_data_path)
print(len(neg_x))
print(len(neg_y))
# print(neg_y)
train_pos_x = pos_x[:41025]
train_pos_y = pos_y[:41025]
val_pos_x = pos_x[41025:52746]
val_pos_y = pos_y[41025:52746]
test_pos_x = pos_x[52746:]
test_pos_y = pos_y[52746:]
train_neg_x = neg_x[:41165]
train_neg_y = neg_y[:41165]
val_neg_x = neg_x[41165:52926]
val_neg_y = neg_y[41165:52926]
test_neg_x = neg_x[52926:]
test_neg_y = neg_y[52926:]
train_x, train_y = concate_data(train_pos_x, train_pos_y, train_neg_x,
train_neg_y)
val_x, val_y = concate_data(val_pos_x, val_pos_y, val_neg_x, val_neg_y)
test_x, test_y = concate_data(test_pos_x, test_pos_y, test_neg_x,
test_neg_y)
print('The number of train-set:', len(train_x))
# print(len(train_y))
print('The number of val-set:', len(val_x))
# print(len(val_y))
print('The number of test-set:', len(test_x))
# print(len(test_y))
embedding = BERTEmbedding('../dataset/chinese_L-12_H-768_A-12',
sequence_length=100)
print('embedding_size', embedding.embedding_size)
# print(embedding.model.output
model = CNNModel(embedding)
model.fit(train_x,
train_y,
val_x,
val_y,
batch_size=128,
epochs=20,
fit_kwargs={'callbacks': [tf_board_callback]})
model.evaluate(test_x, test_y)
model.save('./model/cnn_bert_model')
def eval_models(models_paths: list, path_to_data: str):
if len(models_paths) == 0:
return 0.0
# getting test loader
ds = FashionMnistHandler(path_to_data, False)
ds.download()
ds.load()
# noise parameters are not relevant since test loader shouldn't have noise
_, _, test_loader = ds.get_noisy_loaders(0, '1', 0.2, 128, 128, 128)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
test_acc = []
for model_file in models_paths:
# creating model
checkpoint = torch.load(model_file, map_location=device)
model_name = model_file.split("/")[-1]
# loading from checkpoint
model = CNNModel()
model.load_state_dict(checkpoint['model_state_dict'])
model.to(device)
loss_fn = torch.nn.CrossEntropyLoss()
# evaluating
_, acc = Solver.eval(model,
device,
loss_fn=loss_fn,
data_loader=test_loader)
test_acc.append(acc)
print(f"Model {model_name} has {acc:.4f} acc in test dataset")
return test_acc
def main():
args = sys.argv
batch_size = 128
epochs = 100
maxlen = 300
model_path = 'models/cnn_model.h5'
num_words = 40000
num_label = 2
x, y = load_dataset('data/amazon_reviews_multilingual_JP_v1_00.tsv')
x = preprocess_dataset(x)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
vocab = build_vocabulary(x_train, num_words)
x_train = vocab.texts_to_sequences(x_train)
x_test = vocab.texts_to_sequences(x_test)
x_train = pad_sequences(x_train, maxlen=maxlen, truncating='post')
x_test = pad_sequences(x_test, maxlen=maxlen, truncating='post')
emb_flg = args[0]
if emb_flg == 't':
wv = load_fasttext('../chap08/models/cc.ja.300.vec.gz')
wv = filter_embeddings(wv, vocab.word_index, num_words)
else:
wv = None
model = CNNModel(num_words, num_label, embeddings=wv).build()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
callbacks = [
EarlyStopping(patience=3),
ModelCheckpoint(model_path, save_best_only=True)
]
model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
callbacks=callbacks,
shuffle=True)
model = load_model(model_path)
api = InferenceAPI(model, vocab, preprocess_dataset)
y_pred = api.predict_from_sequences(x_test)
print('precision: {:.4f}'.format(
precision_score(y_test, y_pred, average='binary')))
print('recall : {:.4f}'.format(
recall_score(y_test, y_pred, average='binary')))
print('f1 : {:.4f}'.format(f1_score(y_test, y_pred, average='binary')))
def __init__(self, dm):
self._action_spec = array_spec.BoundedArraySpec(shape=(),
dtype=np.int32,
minimum=0,
maximum=1,
name='action')
self._observation_spec = array_spec.BoundedArraySpec( # !TODO check range of values
shape=(1610, ),
dtype=np.float32,
minimum=0.0,
maximum=1.0,
name='observation')
self._state = np.zeros(
(1610, ), dtype=np.float32
) # observation, should be updated and passes to timestep
self._episode_ended = False
self._n_queries = 0
self.total_reward = 0
self._counter = 0
self.dm = dm
self.cnn_model = CNNModel(self.dm.get_x_shape(), NUM_CLASSES, epochs=5)
self.aux_model = AuxModel(self.cnn_model._model, 4) # Flatten layer
batch_size = 64
num_epochs = 100
####################################################################################
####################### Dataloading ################################################
train_dataset = Fashion_MNIST_Data("./data/train.csv")
trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)
test_dataset = Fashion_MNIST_Data("./data/test.csv")
testloader = DataLoader(test_dataset, batch_size=batch_size,shuffle=False, num_workers=4)
classes = ('T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot')
####################################################################################
####################### Network ####################################################
net = CNNModel()
net.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(),lr=learning_rate)
####################################################################################
####################### Training ###################################################
for epoch in range(num_epochs):
running_loss = 0.0
for i, (images, labels) in enumerate(trainloader):
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
outputs = net(images)
loss = criterion(outputs, torch.max(labels,1)[1])
loss.backward()
def loadModel(target, model_fl):
#model_fl = getBestModelFileName(model_fl)
#print(model_fl)
params = model_fl.split("_")
model_name = params[0]
# print("lo", params)
model = None
#outnode, model_name, target, opt, learn_r, epch, dropout_keep_rate, save_model = False
if model_name == "ImageNetInceptionV2":
model = ImageNetInceptionV2(2, params[0], params[1], params[2],
float(params[3]), int(params[4]),
float(params[5]), False)
else:
model = CNNModel(2, params[0], params[1], params[2], float(params[3]),
int(params[4]), int(params[5]), float(params[6]),
False)
model_fl = model_fl.split(".data")[0]
print("{}/{}".format(model_files_path, model_fl))
model.load("{}/{}".format(model_files_path, model_fl))
chembl_target_threshold_dict = getModelThresholds(
"deepscreen_models_hyperparameters_performance_results.tsv")
compound_smiles_dict = getSMILEsFromFileWithHeader(test_fl)
comp_id_list = list(compound_smiles_dict.keys())
num_of_comps = len(comp_id_list)
#print(num_of_comps)
pred_count = 0
count = 0
print("ACTIVE PREDICTIONS:{}".format(target))
for comp_id in comp_id_list:
count += 1
#print(count)
test_data = []
try:
img_arr = drawPictureandReturnImgMatrix(
TEMP_IMG_OUTPUT_PATH, compound_smiles_dict[comp_id], comp_id)
test_data.append(
[np.array(img_arr / 255.0),
np.array([0, 0]), comp_id])
except:
pass
test_x = np.array([i[0] for i in test_data
]).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
test_comp_name = [i[2] for i in test_data]
test_predictions = model.predict(test_x)
test_predictions = test_predictions[:, 0]
test_pred_labels = [
int(round(i, 2) >= chembl_target_threshold_dict[target])
for i in test_predictions
]
for i in range(len(test_predictions)):
if test_pred_labels[i] == 1:
pred_count += 1
#print("ACTPRED\t{}\t{}\t{}".format(model_fl, target, test_comp_name[i]))
print("{}".format(test_comp_name[i]))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog='cifar10_checkpoint_checker',
description='Script to select best performing checkpoint on CIFAR-10.')
parser.add_argument("checkpoint_dir",
help='Folder containing all .pkl checkpoint files.')
args = parser.parse_args()
checkpoint_dir = args.checkpoint_dir
if not os.path.exists(checkpoint_dir):
raise Exception('Checkpoint directory does not exist.')
checkpoint_list = sorted(os.listdir(checkpoint_dir))
model = CNNModel('xxx', './')
model.fc4.dropout = 0.0
model._compile()
num_channels = model.conv1.filter_shape[0]
filter_size = model.conv1.filter_shape[1]
# Get iterators for cifar10 test set
test_iterator = load_cifar10_data()
# Create object to local contrast normalize a batch.
# Note: Every batch must be normalized before use.
normer = util.Normer2(filter_size=filter_size, num_channels=num_channels)
test_accuracies = []
for i, checkpoint_file in enumerate(checkpoint_list):
class ImageSorter:
def __init__(self, input_path, output_path):
self.input_path = input_path
self.output_path = output_path
set_seed()
if not os.listdir(input_path):
raise NameError("Input directory is empty")
if not os.path.isdir(output_path):
os.mkdir(output_path)
# For speed optimization, all algorithms are defined here
self.backSub = cv.createBackgroundSubtractorMOG2()
self.CNN = CNNModel()
def read_images(self):
data_path = self.input_path + "/*"
files = glob.glob(data_path)
list_of_images = []
for f in files:
current_image = pyplot.imread(f)
list_of_images.append(current_image)
#add some randomness to the data (to remove any bias)
random.shuffle(list_of_images)
return list_of_images
def compare_two_images_L2(self, image_1, image_2):
distance = np.linalg.norm(image_1-image_2)
return distance
def compare_two_images_cosine(self, image_1, image_2):
image_1 = image_1.flatten()
image_2 = image_2.flatten()
distance = cosine(image_1, image_2)
return distance
def apply_CNN(self, list_of_images):
list_of_masks = self.CNN.apply_model(list_of_images)
return list_of_masks
def substract_background(self, list_of_images, background_sub_iterations = 5):
list_of_edited_images = []
for i in range(background_sub_iterations):
for frame in list_of_images:
_ = self.backSub.apply(frame)
for frame in list_of_images:
mask = self.backSub.apply(frame)
new_frame = cv.bitwise_and(frame, frame, mask=mask)
list_of_edited_images.append(new_frame)
return list_of_edited_images
def sort(self, type_of_metric = "L2"):
if(type_of_metric == "L2"):
score = self.compare_two_images_L2
elif(type_of_metric == "cosine"):
score = self.compare_two_images_cosine
else:
raise NameError("Uknown type of metric")
list_of_images = self.read_images()
images_to_save = list_of_images.copy()
list_of_images = self.substract_background(list_of_images)
list_of_images = self.apply_CNN(list_of_images)
list_of_images = list(zip(images_to_save, list_of_images))
main_image = list_of_images.pop()
image_index = 0
image_name = os.path.join(self.output_path, f"image_{image_index}.jpg")
pyplot.imsave(image_name, main_image[0])
while(list_of_images):
current_image = list_of_images[0]
smallest_distance = score(main_image[1], current_image[1])
closest_image_index = 0
for index, current_image in enumerate(list_of_images):
distance = score(main_image[1], current_image[1])
if(distance<smallest_distance):
smallest_distance = distance
closest_image_index = index
main_image = list_of_images.pop(closest_image_index)
image_index = image_index + 1
image_name = os.path.join(self.output_path, f"image_{image_index}.jpg")
pyplot.imsave(image_name, main_image[0])
dtype = torch.float32
device = torch.device("cuda:0")
device2 = torch.device("cuda:1")
seed = 2018
np.random.seed(seed=seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
cudnn.benchmark = True
sub_x, test_data, test_label = get_data()
# oracle_model = Substitute(model=LeNet(), save_path='model/lenet.t7', device=device)
oracle_model = Oracle(model=Rh(num_layers=1, kernel_size=7, n=100000),save_path='model/checkpoints_7_1/ckpt_100000.t7',\
svm_path='model/checkpoints_svm_7_1/svm.pkl', device=device)
if sys.argv[1] == '0' or 'cnn':
sub = CNNModel()
else:
sub = LinearModel()
print(sys.argv[1])
substitute_model = Substitute(model=sub, device=device2)
MNIST_bbox_sub(param=param, oracle_model=oracle_model, substitute_model=substitute_model, \
x_sub=sub_x, test_data=test_data, test_label=test_label, aug_epoch=param['data_aug'],\
samples_max=12800, n_epoch=param['nb_epochs'], fixed_lambda=param['lambda'])
print('\n\nFinal results:')
print('Oracle model evaluation on clean data #%d:' % (test_data.size(0)))
oracle_model.eval(x=test_data, y=test_label, batch_size=10)
print('Substitute model evaluation on clean data: #%d:' %
(test_data.size(0)))
substitute_model.eval(x=test_data, y=test_label, batch_size=512)
def entropy(self, dist):
return -sum([p * np.log(p) for p in dist if p > 0])
if __name__ == '__main__':
config = tf.compat.v1.ConfigProto(device_count={'GPU': 1},
intra_op_parallelism_threads=1,
allow_soft_placement=True)
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.6
session = tf.compat.v1.Session(config=config)
dm, y_oracle = init_dm()
cnn_model = CNNModel(dm.unl.get_feature_shape(), NUM_CLASSES, epochs=5)
env = Environment.create(
environment=CustomEnvironment,
max_episode_timesteps=100,
dm=dm,
session=session,
cnn=cnn_model,
)
agent = Agent.create(
agent='dqn',
states=env.states(),
actions=env.actions(),
batch_size=1,
learning_rate=1e-3,
memory=10000,
loss_name = loss.__class__.__name__
print(f"Loss: {loss_name}\n")
for noise_value in noise_values:
# RUN Experiments
name = f'CNN_{loss_name}_{tp_noise}_{noise_value}'
print(f"Training {name} with noise of type {tp_noise} and probability {noise_value}...")
# data preparation
dataset = FashionMnistHandler(data_dir, False)
dataset.load()
train_loader, val_loader, test_loader = dataset.get_noisy_loaders(p_noise=noise_value,
type_noise=tp_noise,
val_size=1 / 6,
train_batch_size=batch_size,
val_batch_size=128,
test_batch_size=128)
# model, optimizer, summary
model = CNNModel()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
summ = Summary(name, type_noise=tp_noise, noise_rate=noise_value)
solver = Solver(name, PROJECT_DIR, batch_model_dir, batch_summaries_dir, model,
optimizer, loss, summ, train_loader, val_loader, test_loader)
solver.pretrain()
solver.train(loss)
print(f"Completed training...")
help='Training split of stl10 to use. (0-9)')
parser.add_argument(
"checkpoint_dir",
help='Folder containing all .pkl checkpoint files.')
args = parser.parse_args()
train_split = int(args.split)
if train_split < 0 or train_split > 9:
raise Exception('Train split must be in range 0-9.')
checkpoint_dir = args.checkpoint_dir
if not os.path.exists(checkpoint_dir):
raise Exception('Checkpoint directory does not exist.')
checkpoint_list = sorted(os.listdir(checkpoint_dir))
model = CNNModel('xxx', './')
model.fc4.dropout = 0.0
model._compile()
num_channels = model.conv1.filter_shape[0]
filter_size = model.conv1.filter_shape[1]
print 'Using model trained on split '+str(train_split)+'\n'
# Get iterators for stl10 train and test sets
train_iterator, test_iterator = load_stl10_data(train_split)
# Create object to local contrast normalize a batch.
# Note: Every batch must be normalized before use.
normer = util.Normer2(filter_size=filter_size, num_channels=num_channels)
train_accuracies = []
test_accuracies = []
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-2s %(levelname)-5s %(message)s',
datefmt='%y-%m-%d %H:%M',
filemode='w')
action_space = 15
env = WechatJumpEnv(action_space)
gamma = 0.9
epsilon = .95
epsilon_min = 0.01
epsilon_decay = 0.995
learning_rate = 0.01
max_memory = 3000
tau = .125
model = CNNModel((8, 4, 3), (80, 80, 60), (100, 180, 1),
action_space).build_model()
target_model = CNNModel((8, 4, 3), (80, 80, 60), (100, 180, 1),
action_space).build_model()
dqn_agent = DQN(env, model, target_model, epsilon, epsilon_decay,
epsilon_min, gamma, tau, max_memory)
env.reset()
for eposide in range(3000):
cur_state = env.resize_input(env.screen_shot())
action = dqn_agent.choose_act(cur_state)
new_state, reward, done = env.step(action)
dqn_agent.remember(cur_state, action, reward, new_state, done)
dqn_agent.replay()
dqn_agent.target_train() # iterates target model
cur_state = new_state
def trainModelTarget(model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, rotate, save_model):
#target_only_alpnum = ''.join(ch for ch in target if ch.isalnum())
model = None
# ImageNetInceptionv2
# AlexNetModel
# CNNModel2
# CNNModel
if model_name=="ImageNetInceptionV2":
model = ImageNetInceptionV2(2, model_name, target, optimizer, learning_rate, epch, dropout_keep_rate, save_model)
elif model_name == "AlexNetModel":
model = AlexNetModel(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, save_model=False)
elif model_name == "CNNModel":
model = CNNModel(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, dropout_keep_rate, save_model)
elif model_name == "CNNModel2":
model = CNNModel2(2, model_name, target, optimizer, learning_rate, epch, n_of_h1, dropout_keep_rate, save_model)
else:
pass
#train, test = getTrainDataBinary("{}/{}".format(Y_IMG_PATH,target), target )
train, validation, test = constructDataMatricesForMUVDataset(TEMP_IMG_OUTPUT_PATH, target, rotate)
train_comp_name = [i[2] for i in train]
X = []
for i in train:
if i[0].shape!=():
X.append(i[0])
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
Y = []
for i in train:
if i[0].shape != ():
Y.append(i[1])
validation_x = []
for i in validation:
if i[0].shape!=():
validation_x.append(i[0])
validation_x = np.array(validation_x).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
validation_y = []
for i in validation:
if i[0].shape != ():
validation_y.append(i[1])
validation_comp_name = []
for i in validation:
if i[0].shape != ():
validation_comp_name.append(i[2])
test_x = []
for i in test:
if i[0].shape!=():
test_x.append(i[0])
test_x = np.array(test_x).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
test_y = []
for i in test:
if i[0].shape != ():
test_y.append(i[1])
test_comp_name = []
for i in test:
if i[0].shape != ():
test_comp_name.append(i[2])
test_x = np.array([i[0] for i in test]).reshape(-1,IMG_SIZE,IMG_SIZE,1)
test_y = [i[1] for i in test]
test_comp_name = [i[2] for i in test]
if save_model:
model.fit(X, Y, n_epoch=epch, validation_set=({'input': validation_x}, {'targets': validation_y}),
show_metric=True, batch_size=32, snapshot_step=200,
snapshot_epoch=True, run_id="{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_id".format(model_name, target, optimizer, learning_rate, epch, n_of_h1, n_of_h2, dropout_keep_rate, rotate, save_model))
else:
model.fit(X, Y, n_epoch=epch, validation_set=({'input': validation_x}, {'targets': validation_y}),
show_metric=True, batch_size=32)
test_predictions = model.predict(test_x)
validation_predictions = model.predict(validation_x)
test_predictions = test_predictions[:,0]
validation_predictions = validation_predictions[:,0]
#print(validation_predictions)
test_y = [i[0] for i in test_y]
validation_y = [i[0] for i in validation_y]
threshold = 1.00
# f1score, mcc, accuracy, precision, recall, tp, fp, tn, fn, threshold
best_test_accuracy_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
best_test_f1score_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
best_test_mcc_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
best_validation_accuracy_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
best_validation_f1score_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
best_validation_mcc_list = [-1, -1, -1, -1, -1, -1,-1, -1, -1, -1]
validation_auc = roc_auc_score(validation_y, validation_predictions)
validation_auprc = average_precision_score(validation_y, validation_predictions)
# validation_bedroc = bedroc_score(validation_y, validation_predictions)
test_auc = roc_auc_score(test_y, test_predictions)
test_auprc = average_precision_score(test_y, test_predictions)
# test_bedroc = bedroc_score(test_y, test_predictions)
print("Test AUC:{}\nTest AUPRC:{}".format(validation_auc, validation_auprc, test_auc, test_auprc))
while threshold >= 0.00:
validation_pred_labels = [int(round(i,2)>=threshold) for i in validation_predictions]
test_pred_labels = [int(round(i,2)>=threshold) for i in test_predictions]
#print(threshold)
#print(test_pred_labels)
val_precision = precision_score(validation_y, validation_pred_labels)
val_recall = recall_score(validation_y, validation_pred_labels)
val_f1score = f1_score(validation_y, validation_pred_labels)
val_accuracy = accuracy_score(validation_y, validation_pred_labels)
val_mcc = matthews_corrcoef(validation_y, validation_pred_labels)
validation_tn, validation_fp, validation_fn, validation_tp = confusion_matrix(validation_y, validation_pred_labels).ravel()
test_precision = precision_score(test_y, test_pred_labels)
test_recall = recall_score(test_y, test_pred_labels)
test_f1score = f1_score(test_y, test_pred_labels)
test_accuracy = accuracy_score(test_y, test_pred_labels)
test_mcc = matthews_corrcoef(test_y, test_pred_labels)
test_tn, test_fp, test_fn, test_tp = confusion_matrix(test_y, test_pred_labels).ravel()
#print(test_tn, test_fp, test_fn, test_tp)
if val_f1score > best_validation_f1score_list[0]:
best_validation_f1score_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp,
validation_tn, validation_fn,
threshold]
best_test_f1score_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp, test_fp,
test_tn, test_fn,
threshold]
if val_mcc > best_validation_mcc_list[1]:
best_validation_mcc_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp,
validation_tn, validation_fn,
threshold]
best_test_mcc_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp,
test_fp,
test_tn, test_fn,
threshold]
if val_accuracy > best_validation_accuracy_list[2]:
best_validation_accuracy_list = [val_f1score, val_mcc, val_accuracy, val_precision, val_recall, validation_tp, validation_fp,
validation_tn, validation_fn, threshold]
best_test_accuracy_list = [test_f1score, test_mcc, test_accuracy, test_precision, test_recall, test_tp,
test_fp,
test_tn, test_fn,
threshold]
threshold -= 0.01
print(
"Test_f1score:{}\nTest_mcc:{}\nTest_accuracy:{}\nTest_precision:{}\nTest_recall:{}\nTest_tp:{}\nTest_fp:{}\nTest_tn:{}\nTest_fn:{}".format(
round(best_test_mcc_list[0], 2), round(best_test_mcc_list[1], 2), round(best_test_mcc_list[2], 2),
round(best_test_mcc_list[3], 2), round(best_test_mcc_list[4], 2), int(best_test_mcc_list[5]),
int(best_test_mcc_list[6]),
int(best_test_mcc_list[7]),
int(best_test_mcc_list[8])))
best_test_threshold = round(best_test_mcc_list[-1],2)
str_predictions = ""
print("TestPredictions (Threshold:{})".format(best_test_threshold))
for i in range(len(test_predictions)):
temp_pos_pred = round(test_predictions[i], 2)
if test_y[i] == 1 and temp_pos_pred >= best_test_threshold:
str_predictions += "{},{},{}\t".format(test_comp_name[i],"TP","ACT")
elif test_y[i] == 1 and temp_pos_pred < best_test_threshold:
str_predictions += "{},{},{}\t".format(test_comp_name[i], "FN", "ACT")
elif test_y[i] == 0 and temp_pos_pred < best_test_threshold:
str_predictions += "{},{},{}\t".format(test_comp_name[i], "TN", "INACT")
elif test_y[i] == 0 and temp_pos_pred >= best_test_threshold:
str_predictions += "{},{},{}\t".format(test_comp_name[i], "FP", "INACT")
print(str_predictions)
import numpy
from anna import util
from anna.datasets import supervised_dataset
from models import CNNModel
print('Start')
pid = os.getpid()
print('PID: {}'.format(pid))
f = open('pid', 'wb')
f.write(str(pid) + '\n')
f.close()
model = CNNModel('experiment', './', learning_rate=1e-2)
monitor = util.Monitor(model)
# Loading CIFAR-10 dataset
print('Loading Data')
data_path = '/data/cifar10/'
reduced_data_path = os.path.join(data_path, 'reduced', 'cifar10_100')
train_data = numpy.load(os.path.join(reduced_data_path, 'train_X_split_0.npy'))
train_labels = numpy.load(
os.path.join(reduced_data_path, 'train_y_split_0.npy'))
test_data = numpy.load('/data/cifar10/test_X.npy')
test_labels = numpy.load('/data/cifar10/test_y.npy')
train_dataset = supervised_dataset.SupervisedDataset(train_data, train_labels)
test_dataset = supervised_dataset.SupervisedDataset(test_data, test_labels)
)
parser.add_argument("-s", "--split", default="0", help="Training split of stl10 to use. (0-9)")
args = parser.parse_args()
train_split = int(args.split)
if train_split < 0 or train_split > 9:
raise Exception("Training Split must be in range 0-9.")
print("Using STL10 training split: {}".format(train_split))
pid = os.getpid()
print("PID: {}".format(pid))
f = open("pid_" + str(train_split), "wb")
f.write(str(pid) + "\n")
f.close()
model = CNNModel("experiment", "./", learning_rate=1e-2)
monitor = util.Monitor(model, checkpoint_directory="checkpoints_" + str(train_split))
# Loading STL-10 dataset
print("Loading Data")
X_train = numpy.load("/data/stl10_matlab/train_splits/train_X_" + str(train_split) + ".npy")
y_train = numpy.load("/data/stl10_matlab/train_splits/train_y_" + str(train_split) + ".npy")
X_test = numpy.load("/data/stl10_matlab/test_X.npy")
y_test = numpy.load("/data/stl10_matlab/test_y.npy")
X_train = numpy.float32(X_train)
X_train /= 255.0
X_train *= 2.0
X_test = numpy.float32(X_test)
X_test /= 255.0
from anna import util
from anna.datasets import supervised_dataset
import checkpoints
from models import CNNModel
print("Start")
pid = os.getpid()
print("PID: {}".format(pid))
f = open("pid", "wb")
f.write(str(pid) + "\n")
f.close()
model = CNNModel("experiment", "./", learning_rate=1e-2)
checkpoint = checkpoints.unsupervised_layer3
util.set_parameters_from_unsupervised_model(model, checkpoint)
monitor = util.Monitor(model)
# Add dropout
model.fc4.dropout = 0.5
model._compile()
# Loading CIFAR-10 dataset
print("Loading Data")
data_path = "/data/cifar10/"
reduced_data_path = os.path.join(data_path, "reduced", "cifar10_100")
train_data = numpy.load(os.path.join(reduced_data_path, "train_X_split_0.npy"))
train_labels = numpy.load(os.path.join(reduced_data_path, "train_y_split_0.npy"))
def main(args):
print(args)
startime = time.time()
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
# Set hyper-parameters.
batch_size = 128
epochs = 100
maxlen = 300
model_path = 'models/model_{}.h5'
num_words = 40000
num_label = 2
# Data loading.
print(return_time(startime), "1. Loading data ...")
x, y = load_dataset('data/amazon_reviews_multilingual_JP_v1_00.tsv')
# pre-processing.
print(return_time(startime), "2. Preprocessing dataset ...")
x = preprocess_dataset(x)
x_train, x_test, y_train, y_test = train_test_split(x, y,
test_size=0.2,
random_state=42)
vocab = build_vocabulary(x_train, num_words)
x_train = vocab.texts_to_sequences(x_train)
x_test = vocab.texts_to_sequences(x_test)
x_train = pad_sequences(x_train, maxlen=maxlen, truncating='post')
x_test = pad_sequences(x_test, maxlen=maxlen, truncating='post')
# Preparing word embedding.
if args.loadwv:
print(return_time(startime), "3. Loading word embedding ...")
wv_path = 'data/wv_{0}_{1}.npy'.format(maxlen, num_words)
if os.path.exists(wv_path):
wv = np.load(wv_path)
print(return_time(startime), "Loaded word embedding successfully!")
else:
print(return_time(startime), "Word embedding file doesn't exist")
exit()
else:
print(return_time(startime), "3. Preparing word embedding ...")
wv = load_fasttext('data/cc.ja.300.vec.gz')
wv = filter_embeddings(wv, vocab.word_index, num_words)
# Saving word embedding.
if args.savewv:
wv_path = 'data/wv_{0}_{1}.npy'.format(maxlen, num_words)
np.save(wv_path, wv)
print(return_time(startime), "Saved word embedding successfully!", wv_path)
# Build models.
models = [
RNNModel(num_words, num_label, embeddings=None).build(),
LSTMModel(num_words, num_label, embeddings=None).build(),
CNNModel(num_words, num_label, embeddings=None).build(),
RNNModel(num_words, num_label, embeddings=wv).build(),
LSTMModel(num_words, num_label, embeddings=wv).build(),
CNNModel(num_words, num_label, embeddings=wv).build(),
CNNModel(num_words, num_label, embeddings=wv, trainable=False).build()
]
model_names = [
"RNN-None",
"LSTM-None",
"CNN-None",
"RNN-wv",
"LSTM-wv",
"CNN-wv",
"CNN-wv-notrain"
]
print(return_time(startime), "4. Start training ...")
for i, model in enumerate(models):
print("***********************************")
print(return_time(startime), "Model:", model_names[i])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
# Preparing callbacks.
callbacks = [
EarlyStopping(patience=3),
ModelCheckpoint(model_path.format(model_names[i]), save_best_only=True)
]
# Train the model.
model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
validation_split=0.2,
callbacks=callbacks,
shuffle=True)
# Inference.
model = load_model(model_path.format(model_names[i]))
api = InferenceAPI(model, vocab, preprocess_dataset)
y_pred = api.predict_from_sequences(x_test)
print('precision: {:.4f}'.format(precision_score(y_test, y_pred, average='binary')))
print('recall : {:.4f}'.format(recall_score(y_test, y_pred, average='binary')))
print('f1 : {:.4f}'.format(f1_score(y_test, y_pred, average='binary')))