def verify(mat_tf_file,
dataset,
nn_type,
norm_str,
eps,
n_eval,
time_limit,
presolver,
time_limit_bounds=5):
# Example of `mat_tf_file`: model.mat
mat_mip_file = mat_tf_file.replace('.mat',
'_mip.mat') # now: model_mip.mat
out_mip_file_dir = mat_tf_file.replace('.mat', ' eps={}/'.format(eps))
# Convertation of the .mat file to the format supported by MIPVerify.jl
mat_tf = scipy.io.loadmat(mat_tf_file)
mat_mip = convert_mat_tf_to_mip(mat_tf, nn_type)
utils.create_folders([out_mip_file_dir])
scipy.io.savemat(mat_mip_file, mat_mip)
julia_command_to_run = 'julia mip/verify.jl "{}" "{}" {} {} {} {} {} {} {} {}'.format(
mat_mip_file, out_mip_file_dir, dataset, nn_type, norm_str, eps,
n_eval, time_limit, time_limit_bounds, presolver)
print(julia_command_to_run)
os.system(julia_command_to_run)
def update_protein_db():
clean_up_files(utils.INTERMEDIATE_DIR)
proteins = get_protein_sequences()
try:
fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta")
SeqIO.write(proteins, fasta_file, 'fasta')
except:
print "creating proteins folder in correct place"
utils.create_folders()
fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta")
SeqIO.write(proteins, fasta_file, 'fasta')
blast_db_command = [
utils.BLAST_DIR + 'makeblastdb', '-in', "\"" + fasta_file + "\"",
"-dbtype", "prot", "-title", "Proteins", "-out",
"%s" % fasta_file
]
print blast_db_command
# else:
# blast_db_command = [BLAST_DIR + 'formatdb',
# '-i', "\""+ fasta_file+ "\"",
# '-o', 'T',
# "-t", "Proteins"]
# print blast_db_command
subprocess.check_call(blast_db_command)
def before_request():
"""Create data folders at first request time
Args:
None
Returns:
None
"""
if app.config["FIRST_RUN"]:
create_folders()
app.config["FIRST_RUN"] = False
def train(self, train, dev):
saver = tf.train.Saver()
best_accuracy = 0
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
self.add_summary(sess)
for epoch in range(self.config.num_epoch):
self.logger.info("Epoch %d out of %d" %
(epoch + 1, self.config.num_epoch))
accuracy, f05 = self.run(sess, train, dev, epoch)
#self.config.lr *= self.config.lr_decay
if accuracy >= best_accuracy:
best_accuracy = accuracy
create_folders(self.config.model_out)
saver.save(sess, self.config.model_out)
self.logger.info("best accuracy: %f" % (best_accuracy))
def __init__(self,
actions,
get_legal_actions,
transition_and_evaluate,
cuda=torch.cuda.is_available(),
best=False):
utils.create_folders()
self.cuda = cuda
self.actions = actions
self.get_legal_actions = get_legal_actions
self.transition_and_evaluate = transition_and_evaluate
self.models = model_utils.load_models()
self.best_models = model_utils.load_models()
if self.cuda:
activate_cuda(self.models)
activate_cuda(self.best_models)
def __init__(self,
actions,
get_legal_actions,
transition_and_evaluate,
cuda=config.CUDA,
best=False):
utils.create_folders()
self.has_cuda = cuda
self.actions = actions
self.get_legal_actions = get_legal_actions
self.transition_and_evaluate = transition_and_evaluate
self.new = model_utils.load_model(cuda=cuda)
self.best = model_utils.load_model(cuda=cuda)
if self.has_cuda:
self.new = self.new.cuda()
self.best = self.best.cuda()
def save_ranking_one_query(data,
query_desc,
image_names,
path,
image_name,
src_path,
dest_path,
top_image=3):
# data: descriptor_aggregation over 100060 images [100060,512]
# query_desc: descriptor_aggregation over the query image [1,512]
# image_names: list of name of all images
# path: path to save txt file for each query image showing ranking for each image from the dataset of 100700 images
# image_name: name of query image
for i in range(0, image_names.shape[0]):
if image_names[i].replace('\n', '') == image_name:
utils.create_folders(dest_path + image_name + '/')
shutil.copy(src_path + image_name + '.jpg',
dest_path + image_name + '/' + 'A_query_image.jpg')
data_aux = data[i].copy()
data[i] = query_desc
data_local = data
distances, indices = compute_distances_optim(query_desc, data)
sys.stdout.flush()
file = open(path + image_names[i].replace('\n', '') + '.txt', 'w')
for ind in indices:
file.write(image_names[ind])
file.close()
data[i] = data_aux
for j, ind in enumerate(indices):
if j > top_image - 1:
break
try:
shutil.copy(
src_path + image_names[ind].replace('\n', '') + '.jpg',
dest_path + image_name + '/' + 'B_relevant_' +
str(j + 1) + '.jpg')
except:
pass
return indices, data_local
def update_protein_db():
proteins = get_protein_sequences()
try:
fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta")
count = SeqIO.write(proteins, fasta_file, 'fasta')
except:
print "creating proteins folder in correct place"
utils.create_folders()
fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta")
count = SeqIO.write(proteins, fasta_file, 'fasta')
if True:
blast_db_command = [utils.BLAST_DIR + 'makeblastdb',
'-in',"\""+ fasta_file+ "\"",
"-dbtype","prot", "-title", "Proteins",
"-out", "%s"% fasta_file]
print blast_db_command
# else:
# blast_db_command = [BLAST_DIR + 'formatdb',
# '-i', "\""+ fasta_file+ "\"",
# '-o', 'T',
# "-t", "Proteins"]
# print blast_db_command
subprocess.check_call(blast_db_command)
# config variables
weights = config["weights"]
train_path = config["train_path"]
test_path = config["test_path"]
model_path = config["model_path"]
batch_size = config["batch_size"]
epochs = config["epochs"]
classes = config["classes"]
augmented_data = config["augmented_data"]
validation_split = config["validation_split"]
data_augmentation = config["data_augmentation"]
epochs_after_unfreeze = config["epochs_after_unfreeze"]
checkpoint_period = config["checkpoint_period"]
checkpoint_period_after_unfreeze = config["checkpoint_period_after_unfreeze"]
create_folders(model_path, augmented_data)
# create model
if weights == "imagenet":
base_model = MobileNetV2(include_top=False,
weights=weights,
input_tensor=Input(shape=(224, 224, 3)),
input_shape=(224, 224, 3))
top_layers = base_model.output
top_layers = GlobalAveragePooling2D()(top_layers)
top_layers = Dense(1024, activation='relu')(top_layers)
predictions = Dense(classes, activation='softmax')(top_layers)
model = Model(inputs=base_model.input, outputs=predictions)
elif weights == "":
base_model = MobileNetV2(include_top=False,
input_tensor=Input(shape=(224, 224, 3)),
print 'Dataset: ', dataset
print 'Num_cams ', num_cams
print 'PCA with ', num_classes_pca
print 'Model: ', model_name
if do_re_ranking:
print 'Re-ranking with first ', top_n_ranking
if query_expansion:
print 'Applying query expansion using the first ', n_expand
if dataset == 'Oxford':
image_path = '/data/jim011/datasets_retrieval/Oxford5k/images/'
ranking_path = '/flush2/jim011/results/oxford/' + model_name + '/' + dim + '/'
ranking_image_names_list = '../lists/list_oxford_rank.txt'
create_folders(ranking_path)
cam_descriptors_path = '/data/jim011/oxford/descriptors/' + model_name + '/' + dim + '/' + 'oxford_all_64_wp.h5'
pca_descriptors_path = '/data/jim011/paris/descriptors/' + model_name + '/1024x720/' + 'paris_all_64_wp.h5'
t = time.time()
image_names = list()
with open(ranking_image_names_list, "r") as f:
for line in f:
image_names.append(line)
num_images = n_images_oxford
num_img_pca = n_images_paris
num_prec_classes = 64
print 'Dataset: ', dataset
print 'Num_cams ', num_cams
print 'PCA with ', num_classes_pca
if do_re_ranking:
print 'Re-ranking with first ', top_n_ranking
if query_expansion:
print 'Applying query expansion using the first ', n_expand
if dataset == 'Oxford':
image_path = '/data/jim011/datasets_retrieval/Oxford5k/images/'
ranking_path = '../results/oxford/' + model_name + '/' + layer + '/' + dim \
+ '/R' + str(top_n_ranking) + 'QE' + str(n_expand)+'/off/'
ranking_image_names_list = '../lists/list_oxford_rank.txt'
utils.create_folders(ranking_path)
pca_descriptors_path = '/data/jim011/paris/descriptors/Vgg_16_CAM/relu5_1/1024x720/' \
'paris_all_64_wp.h5'
cam_descriptors_path = '/data/jim011/oxford/descriptors/Vgg_16_CAM/relu5_1/1024x720/oxford_all_64_wp.h5'
n_images_pca = n_images_paris
num_images = n_images_oxford
t = time.time()
image_names = list()
with open(ranking_image_names_list, "r") as f:
def __init__(self,
actions,
calculate_reward,
get_legal_actions,
transition,
version=0,
load_model=True,
load_memories=False,
best=False,
trainer=True,
memories=[]):
create_folders()
if memories != []:
self.memories = memories
else:
self.memories = []
self.load_model = load_model
self.load_memories = load_memories
self.actions = actions
self.get_legal_actions = get_legal_actions
self.calculate_reward = calculate_reward
self.transition = transition
self.best = best
self.io = IOStream("checkpoints/run.log")
self.cuda = False
self.models = setup_models(self.io, load_model, self.cuda, trainer)
self.optims = setup_optims(self.models, self.cuda)
self.version = version
if not best:
if load_memories and version is not "best" and memories == []:
print("Loading Memories...")
try:
self.memories = pickle.load(
open("checkpoints/memories.p", "rb"))
except FileNotFoundError:
print("Memories not found, making new memories.")
print("Loading History...")
try:
self.history = pickle.load(open("checkpoints/history.p", "rb"))
except FileNotFoundError:
print("Loss history not found, starting new history.")
self.history = {
"readout": [],
"policy": [],
"value": [],
"total": []
}
self.best_net = MCTSnet(self.actions,
self.calculate_reward,
self.get_legal_actions,
self.transition,
self.version,
self.load_model,
self.load_memories,
best=True,
trainer=False)
])
log.add(
'Epoch: {:d} valid err: {:.3f}% test err: {:.3f}% train err: {:.3f}% avg_d_norm: {:.3f} loss: {:.5f} '
'reg: {:.5f}'.format(epoch, error_rate_valid * 100,
error_rate_test * 100,
error_rate_train * 100, avg_delta_norm,
loss_val - hps.lmbd * reg_val,
hps.lmbd * reg_val))
log.add('Finished this set of hyperparameters in {:.2f} min'.format(
(time.time() - time_start) / 60))
file_name = 'valid={:.2f} test={:.2f} {} '.format(
error_rate_valid * 100, error_rate_test * 100, hps_str)
utils.create_folders([
base_folder + exp_path for base_folder in
['logs', 'models', 'metrics', 'params', 'diffs']
])
log.to_file('logs' + exp_path, file_name) # save optimization output
np.savetxt(
'metrics' + exp_path + file_name,
np.array(metrics)) # save optimization metrics for future plots
saver.save(sess, 'models' + exp_path +
file_name) # save TF model for future real robustness test
if 'mlp' in hps.nn_type:
# (obsolete) We save all parameters to use them in Matlab
utils.save_dicts_of_tf_vars(sess,
model,
exp_path,
file_name,
save_format='mat')
else:
clf = svm.LinearSVC(C=C, random_state=0, max_iter=5000)
clf.fit(X_train, y_train)
y_pred = clf.predict(X_val)
if metric_name == "f1":
metric.append(f1_score(y_val, y_pred, average="micro"))
print("Fold {}: F1 score: {}".format(fold, metric[fold]))
elif metric_name == "uar":
metric.append(recall_score(y_val, y_pred, average="macro") * 100)
print("Fold {}: UAR score: {}".format(fold, metric[fold]))
line.append(str(metric[fold]))
metric_array = np.array(metric)
metric_mean = np.mean(metric_array)
metric_Std = np.std(metric_array)
if metric_name == "f1":
print("Average F1 score: {}".format(metric_mean))
print("Std F1 score: {}".format(metric_Std))
print("--------------------------------------------------------------")
lines.append(";".join(line) + "\n")
elif metric_name == "uar":
print("Average UAR score: {}".format(metric_mean))
print("Std UAR score: {}".format(metric_Std))
print("--------------------------------------------------------------")
lines.append(";".join(line) + "\n")
config_results = get_basic_evaluation("svm", "test", "ds")
create_folders(config_results)
result_file_name = config_results[
'RESULT_PATH'] + "result" + "_ds_" + gender + "_" + args.label_type + ".csv"
with open(result_file_name, "w") as f:
f.writelines(lines)
hps.seed = 1
norm_str_mip = hps.p
hps.p = {'1': 1, '2': 2, 'inf': np.inf}[hps.p]
eps = eps_dict[hps.p][hps.dataset]
hps.gpu_memory = 0.1 if hps.dataset in ['mnist', 'fmnist'] else 0.15
cur_timestamp = str(datetime.now())[:-7] # to get rid of milliseconds
log = utils.Logger()
eval_name = 'eval_{}_l{}'.format(hps.exp_name,
norm_str_mip) # e.g. eval_test_linf
hps.export_folder = 'models_eval/{}/'.format(eval_name)
utils.create_folders([hps.export_folder])
_, x_test, _, y_test = data.get_dataset(hps.dataset)
n_test_ex, hps.height, hps.width, hps.n_col = x_test.shape
hps.n_in, hps.n_out = hps.height * hps.width * hps.n_col, y_test.shape[1]
model_type = hps.nn_type if 'cnn' in hps.nn_type else 'fc' # used to select the correct model from models.py
if hps.nn_type == 'fc1':
hps.n_hs = [1024]
else:
hps.n_hs = []
graph = tf.Graph()
with graph.as_default(), tf.device('/gpu:0'):
x_in = tf.placeholder(tf.float32, [None, hps.height, hps.width, hps.n_col])
y_in = tf.placeholder(tf.float32, [None, hps.n_out])
'--weight_decay',
type=float,
default=1e-16,
metavar='N',
help='clamp the output of the coords function if get too large')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
print(args)
print(args)
torch.manual_seed(args.seed)
device = torch.device("cuda" if args.cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
utils.create_folders(args)
#
dataset = d_selector.retrieve_dataset(args.dataset,
partition="train",
directed=True)
train_loader = Dataloader(dataset, batch_size=1)
dataset = d_selector.retrieve_dataset(args.dataset,
partition="val",
directed=True)
val_loader = Dataloader(dataset, batch_size=1, shuffle=False)
dataset = d_selector.retrieve_dataset(args.dataset,
partition="test",
directed=True)
test_loader = Dataloader(dataset, batch_size=1, shuffle=False)
import torch
import pprint
import os
import pickle as pkl
import time
if __name__ == '__main__':
print(welcome)
time.sleep(2)
parser = get_arguments()
opt = parser.parse_args()
folder_name = create_folders(opt.env)
rewards_list = []
loss_list = []
for loss in ["clipped_loss", "adaptative_KL_loss", "A2C_loss"]:
print("-----------------" + loss + "-----------------")
config = reset_config(opt, print_=False)
if loss == "A2C_loss":
config["batch_size"] = 128
config['epoch'] = 1
config["c2"] = 0
if loss == "clipped_loss":
config['epoch'] = 8
import os
from annotation import AnnotationDataset
from utils import create_folders, load_vocab, \
get_word_vectors, get_feat_vectors, process_word, process_tag
from wimp import WImpModel
from config import config
create_folders(config.model_out)
vocab_words = load_vocab(config.words_vocab_path)
feats = get_feat_vectors(config.feats_file)
word_processor = process_word(vocab_words)
tag_processor = process_tag()
dev = AnnotationDataset(config.dev_data, vocab_words, word_processor,
tag_processor)
test = AnnotationDataset(config.test_data, vocab_words, word_processor,
tag_processor)
train = AnnotationDataset(config.train_data, vocab_words, word_processor,
tag_processor)
model = WImpModel(config, vocab_words, feats)
model.setup()
model.train(train, dev)
model.evaluate(test)
model.interactive_shell(word_processor)
chunk_index = 0
# For saving also features & CAMs
saving_CAMs = False
ind = 0
if dataset == 'distractors100k':
n_img_dataset = 100070
train_list_path_h = "../lists/list_oxford105k_horizontal.txt"
train_list_path_v = "../lists/list_oxford105k_vertical.txt"
path_descriptors = '/data/jim011/distractors100k/descriptors/' + model_name + '/' + layer + '/' + dim + '/'
descriptors_cams_path_wp = path_descriptors + 'distractor_all_' + str(
num_classes) + '_wp'
descriptors_cams_path_mp = path_descriptors + 'distractor_all_' + str(
num_classes) + '_mp'
create_folders(path_descriptors)
# If you want to save features & CAMs
# feature_path = '/imatge/ajimenez/work/ITR/distractors100k/features/' + model_name + '/' + layer + '/' + dim + '/'
# cam_path = '/imatge/ajimenez/work/ITR/distractors100k/cams/' + model_name + '/' + layer + '/' + dim + '/'
# create_folders(feature_path)
# create_folders(cam_path)
def extract_cam_descriptors(model,
batch_size,
num_classes,
size,
mean_value,
image_train_list_path,
desc_wp,
, experiment_name = args.experiment_name+'_'+g
, feature_type = args.feature_type)
# Some other architectures we experimented with but did not make it into the paper
elif args.model_type == 'cnn':
config = get_cnn_config(model_type = args.model_type
, experiment_name = args.experiment_name+'_'+g
, feature_type = args.feature_type)
elif args.model_type == 'cnn_end':
config = get_crnn_config(model_type = args.model_type
, experiment_name = args.experiment_name+'_'+g
, feature_type = args.feature_type)
else:
print("No config for model {} found".format(args.model_type))
exit()
create_folders(config)
data_obj = load_data_object(config, g)
y_pred_folds = {}
y_devel_folds = {}
for parameter in config['parameter_list']:
parameter_text = parameter_str(parameter)
print('[run] ', parameter_text)
y_pred_folds[parameter_text] = []
y_devel_folds[parameter_text] = []
start = datetime.now()
# train and evaluate one model for each fold - 4 splits for training / one hold out
for fold_no in range(len(data_obj.Xs_train)):
from utils import WriteDictToCSV, create_folders, label_map_gen
# Open the train and validation csv file
#df_train = pd.read_csv('data/train_med.csv')
#print "df_train", df_train.shape
#df_valid = pd.read_csv('data/valid_med.csv')
#print "df_valid", df_valid.shape
#df_train = pd.read_csv('data/train_v2.csv')
#print "df_train", df_train.shape
df_valid = pd.read_csv('data/valid_v2.csv')
print "df_valid", df_valid.shape
df_main = pd.read_csv('data/main_v2.csv')
# Generate the dictionary mapping the labels
label_map = label_map_gen(df_main)
print label_map
# Create necessary folders
create_folders()
#Initiate arrays
x_train = []
y_train = []
x_valid = []
y_valid = []
# Process data
data_processing(df_train, x_train, y_train, label_map)
print "test data processed ok"
data_processing(df_valid, x_valid, y_valid, label_map)
print "valid data processed ok"
# datasets/
# ImageNet/
# test_set/
import os
import utils
data_folder = os.path.join(os.getcwd(), 'datasets')
imagenet_folder = os.path.join(data_folder, 'ImageNet')
selected_classes = os.listdir(imagenet_folder)
selected_classes = [x.split('.')[0].split('-')[1] for x in selected_classes]
# Create a separate folder for each class
utils.create_folders([x + '_all' for x in selected_classes], imagenet_folder)
# Save the class labels in a .txt file
with open(os.path.join(imagenet_folder, 'selected_classes.txt'), 'a') as f:
for label in selected_classes:
f.write(label + '\n')
# Uncompress the data into the respective folders
utils.uncompress_folders(imagenet_folder, imagenet_folder)
# Select a few random images from each class,
# and then store them in new folders for the training set.
# Do the same for the test set.
number_of_images = 100
number_of_test_images = 10
utils.select_random_images(selected_classes,
