def verify(mat_tf_file, dataset, nn_type, norm_str, eps, n_eval, time_limit, presolver, time_limit_bounds=5): # Example of `mat_tf_file`: model.mat mat_mip_file = mat_tf_file.replace('.mat', '_mip.mat') # now: model_mip.mat out_mip_file_dir = mat_tf_file.replace('.mat', ' eps={}/'.format(eps)) # Convertation of the .mat file to the format supported by MIPVerify.jl mat_tf = scipy.io.loadmat(mat_tf_file) mat_mip = convert_mat_tf_to_mip(mat_tf, nn_type) utils.create_folders([out_mip_file_dir]) scipy.io.savemat(mat_mip_file, mat_mip) julia_command_to_run = 'julia mip/verify.jl "{}" "{}" {} {} {} {} {} {} {} {}'.format( mat_mip_file, out_mip_file_dir, dataset, nn_type, norm_str, eps, n_eval, time_limit, time_limit_bounds, presolver) print(julia_command_to_run) os.system(julia_command_to_run)
def update_protein_db(): clean_up_files(utils.INTERMEDIATE_DIR) proteins = get_protein_sequences() try: fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta") SeqIO.write(proteins, fasta_file, 'fasta') except: print "creating proteins folder in correct place" utils.create_folders() fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta") SeqIO.write(proteins, fasta_file, 'fasta') blast_db_command = [ utils.BLAST_DIR + 'makeblastdb', '-in', "\"" + fasta_file + "\"", "-dbtype", "prot", "-title", "Proteins", "-out", "%s" % fasta_file ] print blast_db_command # else: # blast_db_command = [BLAST_DIR + 'formatdb', # '-i', "\""+ fasta_file+ "\"", # '-o', 'T', # "-t", "Proteins"] # print blast_db_command subprocess.check_call(blast_db_command)
def before_request(): """Create data folders at first request time Args: None Returns: None """ if app.config["FIRST_RUN"]: create_folders() app.config["FIRST_RUN"] = False
def train(self, train, dev): saver = tf.train.Saver() best_accuracy = 0 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) self.add_summary(sess) for epoch in range(self.config.num_epoch): self.logger.info("Epoch %d out of %d" % (epoch + 1, self.config.num_epoch)) accuracy, f05 = self.run(sess, train, dev, epoch) #self.config.lr *= self.config.lr_decay if accuracy >= best_accuracy: best_accuracy = accuracy create_folders(self.config.model_out) saver.save(sess, self.config.model_out) self.logger.info("best accuracy: %f" % (best_accuracy))
def __init__(self, actions, get_legal_actions, transition_and_evaluate, cuda=torch.cuda.is_available(), best=False): utils.create_folders() self.cuda = cuda self.actions = actions self.get_legal_actions = get_legal_actions self.transition_and_evaluate = transition_and_evaluate self.models = model_utils.load_models() self.best_models = model_utils.load_models() if self.cuda: activate_cuda(self.models) activate_cuda(self.best_models)
def __init__(self, actions, get_legal_actions, transition_and_evaluate, cuda=config.CUDA, best=False): utils.create_folders() self.has_cuda = cuda self.actions = actions self.get_legal_actions = get_legal_actions self.transition_and_evaluate = transition_and_evaluate self.new = model_utils.load_model(cuda=cuda) self.best = model_utils.load_model(cuda=cuda) if self.has_cuda: self.new = self.new.cuda() self.best = self.best.cuda()
def save_ranking_one_query(data, query_desc, image_names, path, image_name, src_path, dest_path, top_image=3): # data: descriptor_aggregation over 100060 images [100060,512] # query_desc: descriptor_aggregation over the query image [1,512] # image_names: list of name of all images # path: path to save txt file for each query image showing ranking for each image from the dataset of 100700 images # image_name: name of query image for i in range(0, image_names.shape[0]): if image_names[i].replace('\n', '') == image_name: utils.create_folders(dest_path + image_name + '/') shutil.copy(src_path + image_name + '.jpg', dest_path + image_name + '/' + 'A_query_image.jpg') data_aux = data[i].copy() data[i] = query_desc data_local = data distances, indices = compute_distances_optim(query_desc, data) sys.stdout.flush() file = open(path + image_names[i].replace('\n', '') + '.txt', 'w') for ind in indices: file.write(image_names[ind]) file.close() data[i] = data_aux for j, ind in enumerate(indices): if j > top_image - 1: break try: shutil.copy( src_path + image_names[ind].replace('\n', '') + '.jpg', dest_path + image_name + '/' + 'B_relevant_' + str(j + 1) + '.jpg') except: pass return indices, data_local
def update_protein_db(): proteins = get_protein_sequences() try: fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta") count = SeqIO.write(proteins, fasta_file, 'fasta') except: print "creating proteins folder in correct place" utils.create_folders() fasta_file = os.path.join(utils.PROTEIN_DB, "Proteins.fasta") count = SeqIO.write(proteins, fasta_file, 'fasta') if True: blast_db_command = [utils.BLAST_DIR + 'makeblastdb', '-in',"\""+ fasta_file+ "\"", "-dbtype","prot", "-title", "Proteins", "-out", "%s"% fasta_file] print blast_db_command # else: # blast_db_command = [BLAST_DIR + 'formatdb', # '-i', "\""+ fasta_file+ "\"", # '-o', 'T', # "-t", "Proteins"] # print blast_db_command subprocess.check_call(blast_db_command)
# config variables weights = config["weights"] train_path = config["train_path"] test_path = config["test_path"] model_path = config["model_path"] batch_size = config["batch_size"] epochs = config["epochs"] classes = config["classes"] augmented_data = config["augmented_data"] validation_split = config["validation_split"] data_augmentation = config["data_augmentation"] epochs_after_unfreeze = config["epochs_after_unfreeze"] checkpoint_period = config["checkpoint_period"] checkpoint_period_after_unfreeze = config["checkpoint_period_after_unfreeze"] create_folders(model_path, augmented_data) # create model if weights == "imagenet": base_model = MobileNetV2(include_top=False, weights=weights, input_tensor=Input(shape=(224, 224, 3)), input_shape=(224, 224, 3)) top_layers = base_model.output top_layers = GlobalAveragePooling2D()(top_layers) top_layers = Dense(1024, activation='relu')(top_layers) predictions = Dense(classes, activation='softmax')(top_layers) model = Model(inputs=base_model.input, outputs=predictions) elif weights == "": base_model = MobileNetV2(include_top=False, input_tensor=Input(shape=(224, 224, 3)),
print 'Dataset: ', dataset print 'Num_cams ', num_cams print 'PCA with ', num_classes_pca print 'Model: ', model_name if do_re_ranking: print 'Re-ranking with first ', top_n_ranking if query_expansion: print 'Applying query expansion using the first ', n_expand if dataset == 'Oxford': image_path = '/data/jim011/datasets_retrieval/Oxford5k/images/' ranking_path = '/flush2/jim011/results/oxford/' + model_name + '/' + dim + '/' ranking_image_names_list = '../lists/list_oxford_rank.txt' create_folders(ranking_path) cam_descriptors_path = '/data/jim011/oxford/descriptors/' + model_name + '/' + dim + '/' + 'oxford_all_64_wp.h5' pca_descriptors_path = '/data/jim011/paris/descriptors/' + model_name + '/1024x720/' + 'paris_all_64_wp.h5' t = time.time() image_names = list() with open(ranking_image_names_list, "r") as f: for line in f: image_names.append(line) num_images = n_images_oxford num_img_pca = n_images_paris
num_prec_classes = 64 print 'Dataset: ', dataset print 'Num_cams ', num_cams print 'PCA with ', num_classes_pca if do_re_ranking: print 'Re-ranking with first ', top_n_ranking if query_expansion: print 'Applying query expansion using the first ', n_expand if dataset == 'Oxford': image_path = '/data/jim011/datasets_retrieval/Oxford5k/images/' ranking_path = '../results/oxford/' + model_name + '/' + layer + '/' + dim \ + '/R' + str(top_n_ranking) + 'QE' + str(n_expand)+'/off/' ranking_image_names_list = '../lists/list_oxford_rank.txt' utils.create_folders(ranking_path) pca_descriptors_path = '/data/jim011/paris/descriptors/Vgg_16_CAM/relu5_1/1024x720/' \ 'paris_all_64_wp.h5' cam_descriptors_path = '/data/jim011/oxford/descriptors/Vgg_16_CAM/relu5_1/1024x720/oxford_all_64_wp.h5' n_images_pca = n_images_paris num_images = n_images_oxford t = time.time() image_names = list() with open(ranking_image_names_list, "r") as f:
def __init__(self, actions, calculate_reward, get_legal_actions, transition, version=0, load_model=True, load_memories=False, best=False, trainer=True, memories=[]): create_folders() if memories != []: self.memories = memories else: self.memories = [] self.load_model = load_model self.load_memories = load_memories self.actions = actions self.get_legal_actions = get_legal_actions self.calculate_reward = calculate_reward self.transition = transition self.best = best self.io = IOStream("checkpoints/run.log") self.cuda = False self.models = setup_models(self.io, load_model, self.cuda, trainer) self.optims = setup_optims(self.models, self.cuda) self.version = version if not best: if load_memories and version is not "best" and memories == []: print("Loading Memories...") try: self.memories = pickle.load( open("checkpoints/memories.p", "rb")) except FileNotFoundError: print("Memories not found, making new memories.") print("Loading History...") try: self.history = pickle.load(open("checkpoints/history.p", "rb")) except FileNotFoundError: print("Loss history not found, starting new history.") self.history = { "readout": [], "policy": [], "value": [], "total": [] } self.best_net = MCTSnet(self.actions, self.calculate_reward, self.get_legal_actions, self.transition, self.version, self.load_model, self.load_memories, best=True, trainer=False)
]) log.add( 'Epoch: {:d} valid err: {:.3f}% test err: {:.3f}% train err: {:.3f}% avg_d_norm: {:.3f} loss: {:.5f} ' 'reg: {:.5f}'.format(epoch, error_rate_valid * 100, error_rate_test * 100, error_rate_train * 100, avg_delta_norm, loss_val - hps.lmbd * reg_val, hps.lmbd * reg_val)) log.add('Finished this set of hyperparameters in {:.2f} min'.format( (time.time() - time_start) / 60)) file_name = 'valid={:.2f} test={:.2f} {} '.format( error_rate_valid * 100, error_rate_test * 100, hps_str) utils.create_folders([ base_folder + exp_path for base_folder in ['logs', 'models', 'metrics', 'params', 'diffs'] ]) log.to_file('logs' + exp_path, file_name) # save optimization output np.savetxt( 'metrics' + exp_path + file_name, np.array(metrics)) # save optimization metrics for future plots saver.save(sess, 'models' + exp_path + file_name) # save TF model for future real robustness test if 'mlp' in hps.nn_type: # (obsolete) We save all parameters to use them in Matlab utils.save_dicts_of_tf_vars(sess, model, exp_path, file_name, save_format='mat') else:
clf = svm.LinearSVC(C=C, random_state=0, max_iter=5000) clf.fit(X_train, y_train) y_pred = clf.predict(X_val) if metric_name == "f1": metric.append(f1_score(y_val, y_pred, average="micro")) print("Fold {}: F1 score: {}".format(fold, metric[fold])) elif metric_name == "uar": metric.append(recall_score(y_val, y_pred, average="macro") * 100) print("Fold {}: UAR score: {}".format(fold, metric[fold])) line.append(str(metric[fold])) metric_array = np.array(metric) metric_mean = np.mean(metric_array) metric_Std = np.std(metric_array) if metric_name == "f1": print("Average F1 score: {}".format(metric_mean)) print("Std F1 score: {}".format(metric_Std)) print("--------------------------------------------------------------") lines.append(";".join(line) + "\n") elif metric_name == "uar": print("Average UAR score: {}".format(metric_mean)) print("Std UAR score: {}".format(metric_Std)) print("--------------------------------------------------------------") lines.append(";".join(line) + "\n") config_results = get_basic_evaluation("svm", "test", "ds") create_folders(config_results) result_file_name = config_results[ 'RESULT_PATH'] + "result" + "_ds_" + gender + "_" + args.label_type + ".csv" with open(result_file_name, "w") as f: f.writelines(lines)
hps.seed = 1 norm_str_mip = hps.p hps.p = {'1': 1, '2': 2, 'inf': np.inf}[hps.p] eps = eps_dict[hps.p][hps.dataset] hps.gpu_memory = 0.1 if hps.dataset in ['mnist', 'fmnist'] else 0.15 cur_timestamp = str(datetime.now())[:-7] # to get rid of milliseconds log = utils.Logger() eval_name = 'eval_{}_l{}'.format(hps.exp_name, norm_str_mip) # e.g. eval_test_linf hps.export_folder = 'models_eval/{}/'.format(eval_name) utils.create_folders([hps.export_folder]) _, x_test, _, y_test = data.get_dataset(hps.dataset) n_test_ex, hps.height, hps.width, hps.n_col = x_test.shape hps.n_in, hps.n_out = hps.height * hps.width * hps.n_col, y_test.shape[1] model_type = hps.nn_type if 'cnn' in hps.nn_type else 'fc' # used to select the correct model from models.py if hps.nn_type == 'fc1': hps.n_hs = [1024] else: hps.n_hs = [] graph = tf.Graph() with graph.as_default(), tf.device('/gpu:0'): x_in = tf.placeholder(tf.float32, [None, hps.height, hps.width, hps.n_col]) y_in = tf.placeholder(tf.float32, [None, hps.n_out])
'--weight_decay', type=float, default=1e-16, metavar='N', help='clamp the output of the coords function if get too large') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() print(args) print(args) torch.manual_seed(args.seed) device = torch.device("cuda" if args.cuda else "cpu") kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} utils.create_folders(args) # dataset = d_selector.retrieve_dataset(args.dataset, partition="train", directed=True) train_loader = Dataloader(dataset, batch_size=1) dataset = d_selector.retrieve_dataset(args.dataset, partition="val", directed=True) val_loader = Dataloader(dataset, batch_size=1, shuffle=False) dataset = d_selector.retrieve_dataset(args.dataset, partition="test", directed=True) test_loader = Dataloader(dataset, batch_size=1, shuffle=False)
import torch import pprint import os import pickle as pkl import time if __name__ == '__main__': print(welcome) time.sleep(2) parser = get_arguments() opt = parser.parse_args() folder_name = create_folders(opt.env) rewards_list = [] loss_list = [] for loss in ["clipped_loss", "adaptative_KL_loss", "A2C_loss"]: print("-----------------" + loss + "-----------------") config = reset_config(opt, print_=False) if loss == "A2C_loss": config["batch_size"] = 128 config['epoch'] = 1 config["c2"] = 0 if loss == "clipped_loss": config['epoch'] = 8
import os from annotation import AnnotationDataset from utils import create_folders, load_vocab, \ get_word_vectors, get_feat_vectors, process_word, process_tag from wimp import WImpModel from config import config create_folders(config.model_out) vocab_words = load_vocab(config.words_vocab_path) feats = get_feat_vectors(config.feats_file) word_processor = process_word(vocab_words) tag_processor = process_tag() dev = AnnotationDataset(config.dev_data, vocab_words, word_processor, tag_processor) test = AnnotationDataset(config.test_data, vocab_words, word_processor, tag_processor) train = AnnotationDataset(config.train_data, vocab_words, word_processor, tag_processor) model = WImpModel(config, vocab_words, feats) model.setup() model.train(train, dev) model.evaluate(test) model.interactive_shell(word_processor)
chunk_index = 0 # For saving also features & CAMs saving_CAMs = False ind = 0 if dataset == 'distractors100k': n_img_dataset = 100070 train_list_path_h = "../lists/list_oxford105k_horizontal.txt" train_list_path_v = "../lists/list_oxford105k_vertical.txt" path_descriptors = '/data/jim011/distractors100k/descriptors/' + model_name + '/' + layer + '/' + dim + '/' descriptors_cams_path_wp = path_descriptors + 'distractor_all_' + str( num_classes) + '_wp' descriptors_cams_path_mp = path_descriptors + 'distractor_all_' + str( num_classes) + '_mp' create_folders(path_descriptors) # If you want to save features & CAMs # feature_path = '/imatge/ajimenez/work/ITR/distractors100k/features/' + model_name + '/' + layer + '/' + dim + '/' # cam_path = '/imatge/ajimenez/work/ITR/distractors100k/cams/' + model_name + '/' + layer + '/' + dim + '/' # create_folders(feature_path) # create_folders(cam_path) def extract_cam_descriptors(model, batch_size, num_classes, size, mean_value, image_train_list_path, desc_wp,
, experiment_name = args.experiment_name+'_'+g , feature_type = args.feature_type) # Some other architectures we experimented with but did not make it into the paper elif args.model_type == 'cnn': config = get_cnn_config(model_type = args.model_type , experiment_name = args.experiment_name+'_'+g , feature_type = args.feature_type) elif args.model_type == 'cnn_end': config = get_crnn_config(model_type = args.model_type , experiment_name = args.experiment_name+'_'+g , feature_type = args.feature_type) else: print("No config for model {} found".format(args.model_type)) exit() create_folders(config) data_obj = load_data_object(config, g) y_pred_folds = {} y_devel_folds = {} for parameter in config['parameter_list']: parameter_text = parameter_str(parameter) print('[run] ', parameter_text) y_pred_folds[parameter_text] = [] y_devel_folds[parameter_text] = [] start = datetime.now() # train and evaluate one model for each fold - 4 splits for training / one hold out for fold_no in range(len(data_obj.Xs_train)):
from utils import WriteDictToCSV, create_folders, label_map_gen # Open the train and validation csv file #df_train = pd.read_csv('data/train_med.csv') #print "df_train", df_train.shape #df_valid = pd.read_csv('data/valid_med.csv') #print "df_valid", df_valid.shape #df_train = pd.read_csv('data/train_v2.csv') #print "df_train", df_train.shape df_valid = pd.read_csv('data/valid_v2.csv') print "df_valid", df_valid.shape df_main = pd.read_csv('data/main_v2.csv') # Generate the dictionary mapping the labels label_map = label_map_gen(df_main) print label_map # Create necessary folders create_folders() #Initiate arrays x_train = [] y_train = [] x_valid = [] y_valid = [] # Process data data_processing(df_train, x_train, y_train, label_map) print "test data processed ok" data_processing(df_valid, x_valid, y_valid, label_map) print "valid data processed ok"
# datasets/ # ImageNet/ # test_set/ import os import utils data_folder = os.path.join(os.getcwd(), 'datasets') imagenet_folder = os.path.join(data_folder, 'ImageNet') selected_classes = os.listdir(imagenet_folder) selected_classes = [x.split('.')[0].split('-')[1] for x in selected_classes] # Create a separate folder for each class utils.create_folders([x + '_all' for x in selected_classes], imagenet_folder) # Save the class labels in a .txt file with open(os.path.join(imagenet_folder, 'selected_classes.txt'), 'a') as f: for label in selected_classes: f.write(label + '\n') # Uncompress the data into the respective folders utils.uncompress_folders(imagenet_folder, imagenet_folder) # Select a few random images from each class, # and then store them in new folders for the training set. # Do the same for the test set. number_of_images = 100 number_of_test_images = 10 utils.select_random_images(selected_classes,