def init(): print 'Loading training samples..' training_samples = utils.load_samples('../data/askubuntu/train_random.txt') print len(training_samples) print 'Loading dev samples..' dev_samples = utils.load_samples('../data/askubuntu/dev.txt') print len(dev_samples) print 'Loading test samples..' test_samples = utils.load_samples('../data/askubuntu/test.txt') print len(test_samples) print 'Loading corpus..' question_map = utils.load_corpus('../data/askubuntu/text_tokenized.txt') print len(question_map) print 'Loading stop words..' stop_words = utils.load_stop_words('../data/english_stop_words.txt') print len(stop_words) corpus_texts = map(lambda (t, b): t + ' ' + b, question_map.values()) print 'Loading embeddings..' embedding_map = utils.load_embeddings( '../data/pruned_askubuntu_android_vector.txt', corpus_texts, stop_words) print len(embedding_map) print utils.store_embedding_map(embedding_map) return (training_samples, dev_samples, test_samples, question_map, embedding_map)
def init(): print 'Loading askubuntu training samples..' askubuntu_training_samples = utils.load_samples( '../data/askubuntu/train_random.txt') print len(askubuntu_training_samples) print 'Loading askubuntu dev samples..' askubuntu_dev_samples = utils.load_samples('../data/askubuntu/dev.txt') print len(askubuntu_dev_samples) print 'Loading askubuntu test samples..' askubuntu_test_samples = utils.load_samples('../data/askubuntu/test.txt') print len(askubuntu_test_samples) print 'Loading askubuntu corpus..' askubuntu_question_map = utils.load_corpus( '../data/askubuntu/text_tokenized.txt') print len(askubuntu_question_map) print 'Loading android dev samples..' android_dev_samples = utils.load_samples_stupid_format( '../data/android/dev.pos.txt', '../data/android/dev.neg.txt') print len(android_dev_samples) print 'Loading android test samples..' android_test_samples = utils.load_samples_stupid_format( '../data/android/test.pos.txt', '../data/android/test.neg.txt') print len(android_test_samples) print 'Loading android corpus..' android_question_map = utils.load_corpus('../data/android/corpus.tsv') print len(android_question_map) print 'Loading stop words..' stop_words = utils.load_stop_words('../data/english_stop_words.txt') print len(stop_words) corpus_texts = map(lambda (t, b): t + ' ' + b, askubuntu_question_map.values() + android_question_map.values()) print 'Loading embeddings..' embedding_map = utils.load_embeddings( '../data/pruned_android_vector.txt', corpus_texts, stop_words) # pruned_askubuntu_android_vector.txt print len(embedding_map) print utils.store_embedding_map(embedding_map) return ( askubuntu_training_samples, askubuntu_dev_samples, askubuntu_test_samples, askubuntu_question_map, android_dev_samples, android_test_samples, android_question_map, embedding_map)
def validate_decoder(target_path, generation_path): from numpy import linalg as LA import numpy as np target_info, targets, _ = load_samples(target_path) generate_info, generates, _ = load_samples(generation_path) if len(targets) != len(generates): return False for i in range(len(targets)): if LA.norm(np.subtract(targets[i], generates[i])) > 0.001: return False return True
def main(args): if args.get('verbose', False): print(args) recovery_method_name = args["recovery_method"] recovery_params = args["recovery_params"] RecoveryMethodClass = getattr(recovery, recovery_method_name) graph = args.get("graph") if graph is None: graph = load_graph(args["graph_file"]) samples = args.get("samples") if samples is None: samples = load_samples(args["samples_file"]) recovery_method = RecoveryMethodClass(graph, samples, recovery_params) x = [graph.node[idx]['value'] for idx in sorted(graph.node)] x_hat = recovery_method.run() results = args.copy() results.update({"x_hat": x_hat, "nmse": nmse(x, x_hat)}) results_file = args.get("results_file") if results_file is None: return results else: dump_results(results, results_file)
def train_network(opts): # load data action_info, action_data, target_data = load_samples( opts.data_path, options.dirty_sample_path) # construct model if 'AE' == opts.model: model, encoder, decoder = ConvAE(opts.nfs, opts.sks, opts.nz, opts.input_size, opts.dropout) # elif 'VAE' == opts.model: # model = ConvVAE(opts.nfs, opts.sks, opts.nz, opts.input_size) # model.compile(optimizer='rmsprop', loss=vae_loss) # model.fit(inputs=[action_data, target_data], epochs=opts.epochs, batch_size=opts.batch_size, # callbacks=[tbCallBack, BestLossCallBack(model, opts.save_path, opts.save_period)], shuffle=True) else: print('There is no proper model for option: ' + opts.model) return callback_list = [ tbCallBack, BestLossCallBack(model, opts.save_path, opts.save_period) ] # BestLossCallBack(decoder, opts.save_path, opts.save_period, 'decoder') # training model.compile(optimizer='rmsprop', loss='mse') model.fit(action_data, target_data, epochs=opts.epochs, batch_size=opts.batch_size, callbacks=callback_list, shuffle=True)
def load_data(prob_folder, logfile): files = list(pathlib.Path(prob_folder).glob('sample_*.pkl'))[:100] xs, ys, cands = load_samples(files, logfile) x_shift = xs.mean(axis=0) x_scale = xs.std(axis=0) x_scale[x_scale == 0] = 1 xs = (xs - x_shift) / x_scale return xs, ys, cands
def instance(request): if request.method == 'POST': request_data = json.loads(request.body) data_index = request_data['dataIndex'] data, _ = load_samples(dataset=rs['dataset'], client_name=rs['client'], sampling_type='local') return JsonResponse({'data': data[data_index].tolist()})
def load(begin, to): # Data loading xs, ys, cands = load_samples(files[begin:to], logfile) log(f" {xs.shape[0]} training samples", logfile) # Data normalization x_shift = xs.mean(axis=0) x_scale = xs.std(axis=0) x_scale[x_scale == 0] = 1 xs = (xs - x_shift) / x_scale return xs, ys, cands
def attribute(request): if request.method == 'POST': request_data = json.loads(request.body) dim_index = request_data['dimIndex'] data, _ = load_samples(dataset=rs['dataset'], client_name=rs['client'], sampling_type='local') attr_data = data[:, dim_index] return JsonResponse({'data': attr_data.tolist()})
def sampling(request): if request.method == 'POST': request_data = json.loads(request.body) sampling_type = request_data['samplingType'] samples, ground_truth = load_samples(dataset=rs.state['dataset'], client_name=rs.state['client'], sampling_type=sampling_type) rs.set('data', samples) rs.set('sampling_type', sampling_type) rs.set('ground_truth', ground_truth) return JsonResponse({})
def cpca_cluster(request): if request.method == 'POST': request_data = json.loads(request.body) alpha = None if 'alpha' in request_data.keys(): alpha = request_data['alpha'] bg_data_idx = request_data['dataIndex'] data = rs.state['data'] homo_idx = rs.state['outputs_server'] == rs.state['outputs_client'] bg = data[bg_data_idx] if bg.shape[0] == 1: return JsonResponse(rs.state['cpca_all_result']) fg = np.concatenate((data[homo_idx], bg)) local_data, _ = load_samples(dataset=rs['dataset'], client_name=rs['client'], sampling_type='local') if USE_GPU: with torch.no_grad(): cPCA = CPCA_GPU(n_components=2) cPCA.fit(target=fg, background=bg, alpha=alpha) projected_data = cPCA.transform(local_data) components = cPCA.get_components() alpha = cPCA.get_alpha() else: cPCA = CPCA(n_components=2) cPCA.fit(target=fg, background=bg, alpha=alpha) projected_data = cPCA.transform(local_data) components = cPCA.get_components() alpha = cPCA.get_alpha() projected_data = np.round(projected_data, 6).astype(float) components = np.round(components, 6).astype(float) alpha = np.round(alpha, 6).astype(float) data = { 'alpha': alpha, 'cPC1': components[0].tolist(), 'cPC2': components[1].tolist(), 'projectedData': projected_data.tolist(), } return JsonResponse(data)
def compute_cov(state): """Computes the crosspectrum matrices per subjects.""" for sub in SUBJECT_LIST: pattern = prefix + "_s{}_{}.mat" file_path = path(SAVE_PATH / pattern.format(sub, state)) if not file_path.isfile(): # data must be of shape n_trials x n_elec x n_samples data = load_samples(DATA_PATH, sub, state) if FULL_TRIAL: data = np.concatenate(data, axis=1) data = data.reshape(1, data.shape[0], data.shape[1]) cov = Covariances() mat = cov.fit_transform(data) savemat(file_path, {"data": mat})
def __init__(self, graph, samples): super().__init__() if isinstance(graph, nx.Graph): self.graph = graph elif isinstance(graph, str): self.graph = load_graph(graph) else: raise ValueError("unexpected graph type: {}".format(type(graph))) if isinstance(samples, (list, tuple, set)): self.samples = samples elif isinstance(samples, str): self.samples = load_samples(samples) else: raise ValueError("unexpected samples type: {}".format(type(samples)))
def computeAndSavePSD( SUBJECT_LIST, state, freq, window, overlap, fmin, fmax, fs, elec=None ): """loads data, compute PSD and saves PSD of all subjects in one file""" N_ELEC = 19 if elec is None else len(elec) print(state, freq, "bande {}: [{}-{}]Hz".format(freq, fmin, fmax)) for elec in range(N_ELEC): # pour chaque elec channel_name = CHANNEL_NAMES[elec] file_path = path( SAVE_PATH / "PSD_{}_{}_{}_{}_{:.2f}.mat".format( # 'PSD_EOG_sleepState_%s_%s_%i_%i_%.2f.mat' % state, freq, channel_name, window, overlap, ) ) if not file_path.isfile(): psds = [] for sub in SUBJECT_LIST: # pour chaque sujet X = load_samples(DATA_PATH, sub, state) psd_list = [] for j in range(X.shape[0]): # pour chaque trial psd = computePSD( X[j, elec], window=window, overlap=OVERLAP, fmin=fmin, fmax=fmax, fs=fs, ) psd_list.append(psd) psd_list = np.asarray(psd_list) psds.append(psd_list.ravel()) print(file_path) savemat(file_path, {"data": psds})
def main(argv): # creating the algorithms. the optimal hyper-parameters are hardcoded. per = Perceptron() svm = SVM() pa = PA() # loading the train set, including the labels on the rightmost column. train_set = utils.load_dset(argv[0], argv[1]) # loading the test set. add a right column of ones for the bias test_x = utils.load_samples(argv[2]) # normalizing the train and the test sets with min-max norm. min = 4, max = 30. mins, denoms = norm.minmax_params(train_set) train_set = norm.minmax(train_set, 4, 30) test_x = norm.minmax(test_x, 4, 30, mins, denoms) # training the algorithms . train(train_set, per, svm, pa, 100) # predicting and printing the results on the test set. predict(test_x, per, svm, pa)
def compute_cosp(state, freq, window, overlap, cycle=None): """Computes the crosspectrum matrices per subjects.""" if cycle is not None: print(state, freq, cycle) else: print(state, freq) freqs = FREQ_DICT[freq] for sub in SUBJECT_LIST: if cycle is None: file_path = SAVE_PATH / prefix + "_s{}_{}_{}_{}_{:.2f}.mat".format( sub, state, freq, window, overlap) else: file_path = SAVE_PATH / prefix + "_s{}_{}_cycle{}_{}_{}_{:.2f}.mat".format( sub, state, cycle, freq, window, overlap) if not file_path.isfile(): # data must be of shape n_trials x n_elec x n_samples if cycle is not None: data = load_full_sleep(DATA_PATH, sub, state, cycle) if data is None: continue data = data.swapaxes(1, 2) else: data = load_samples(DATA_PATH, sub, state) if FULL_TRIAL: data = np.concatenate(data, axis=1) data = data.reshape(1, data.shape[0], data.shape[1]) cov = CospCovariances(window=window, overlap=overlap, fmin=freqs[0], fmax=freqs[1], fs=SF) mat = cov.fit_transform(data) if len(mat.shape) > 3: mat = np.mean(mat, axis=-1) savemat(file_path, {"data": mat})
from sklearn.model_selection import train_test_split from sklearn.utils import shuffle from keras.models import Sequential from keras.layers import Lambda, Flatten, Dense, Cropping2D, Convolution2D, MaxPooling2D, Dropout from keras.optimizers import Adam from keras.regularizers import l2 from keras.callbacks import ModelCheckpoint, Callback from keras.utils.visualize_util import plot import utils import constants as CONST # LOAD SAMPLES: # Load all samples from one or multiple files. all_samples = utils.load_samples(CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE, CONST.BEACH_DATA_FILES) # TRAIN/VALIDATION SPLITS: # Split all the samples in training and validation and calculate the number of samples in each set taking into account # the criteria and methods that will be used to augment them. train_samples, validation_samples = train_test_split(all_samples, test_size=CONST.TEST_SIZE) TRAIN_SAMPLES_COUNT = utils.calculate_augmented_size(train_samples, CONST.ANGLE_CORRECTION, CONST.SKIP_FILTER) VALIDATION_SAMPLES_COUNT = utils.calculate_augmented_size( validation_samples, CONST.ANGLE_CORRECTION, CONST.SKIP_FILTER) # IMAGE GENERATOR:
from sklearn.model_selection import train_test_split from sklearn.linear_model import SGDClassifier from utils import load_samples, count_files, evaluate, imagenet_preprocess_input, _set_n_retrain, _get_weighted_layers, producer, getPaths, load_data, load_data_part, categorical_acc, load_data_frac from networks import simple_gram, resnet_trained #Must modify the paths so that they point to your local wikipaintings files data = "wikiart/wikiart/" data_train = r"C:\Users\tgill\OneDrive\Documents\GD_AI\ArtGAN\wikipaintings_full\wikipaintings_train" data_test = r"C:\Users\tgill\OneDrive\Documents\GD_AI\ArtGAN\wikipaintings_full\wikipaintings_val" target_size = (224, 224) X, y, classes = load_samples(10) X_test, y_test, classes_test = load_samples(10, data_test) #m = resnet_trained(20) m = simple_gram(20) print(m.summary()) n_files_train = count_files(data_train) n_files_test = count_files(data_test) nepochs = 20 epoch_size = 2500 batch_size = 32 steps_per_epoch = (n_files_train//batch_size) v_step = 50 #n_files_test//batch_size
import matplotlib.pyplot as plt import math from sklearn.model_selection import train_test_split import utils import constants as CONST # LOAD DATA: # Track 1 (Beach): samples_beach = utils.load_samples( CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE, [ CONST.BEACH_4_ANTICLOCK_FILE, CONST.BEACH_4_CLOCK_FILE, CONST.RECO_BEACH_1_ANTICLOCK_FILE, CONST.RECO_BEACH_1_CLOCK_FILE ]) # Track 2 (Mountain): samples_mountain = utils.load_samples( CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE, [ CONST.MOUNTAIN_4_ANTICLOCK_FILE, CONST.MOUNTAIN_4_CLOCK_FILE, CONST.RECO_MOUNTAIN_1_ANTICLOCK_FILE, CONST.RECO_MOUNTAIN_1_CLOCK_FILE ]) # EXTRACT BASE ANGLES: angles_beach = list(map(lambda sample: sample["steering"], samples_beach)) angles_mountain = list(map(lambda sample: sample["steering"], samples_mountain))
import numpy as np import progressbar import os import re import glob from utils import load_samples, save_samples kDataRoot = "C:/Users/JM/Desktop/Data/ETRIrelated/BMVC" kActionRoot = os.path.join(kDataRoot, "etri_action_data_neck_point_0_0") if __name__ == "__main__": all_data, _, file_names = load_samples(kActionRoot)
f1 = 2 * precision * recall / (precision + recall) f1s_list.append(f1) except ZeroDivisionError: f1s_list.append(np.nan) return list(mse_grid)[int(np.nanargmax(f1s_list))] if __name__ == "__main__": for k in range(10): # validation step for find threshold value video_list = cv_file_list(k, 'validation') # k = 1 data_info, action_data, _ = load_samples(kActionData) val_data = [] val_info = [] for i, info in enumerate(data_info): video_name = str(int(info['video_name'])) if video_name in video_list: val_data.append(action_data[i]) val_info.append(info) val_data = np.array(val_data) # TODO: run model for make validation error model_name = os.path.join(kDetectBase, "best_loss.hdf5") model = load_model(model_name) predictions = model.predict(val_data) save_recon_error(kDetectBase, val_data, predictions, val_info, "recon_val_error.csv")
from discriminator import Discriminator from generator import Generator from parameters import d_conv_dim, z_size, g_conv_dim, beta1, beta2, lr_d, lr_g from train import train from utils import display_images, weights_init_normal, gpu_check, load_samples, view_samples display_images(loader) D = Discriminator(d_conv_dim) G = Generator(z_size=z_size, conv_dim=g_conv_dim) D.apply(weights_init_normal) G.apply(weights_init_normal) train_on_gpu = gpu_check() if not train_on_gpu: print('No GPU found. Please use a GPU to train your neural network.') else: print('Training on GPU!') d_optimizer = optim.Adam(D.parameters(), lr_d, [beta1, beta2]) g_optimizer = optim.Adam(G.parameters(), lr_g, [beta1, beta2]) n_epochs = 30 losses = train(D, d_optimizer, G, g_optimizer, n_epochs=n_epochs) samples = load_samples() _ = view_samples(-1, samples)
from batch_generator import BatchGenerator from callbacks import SaveEmbeddingModel from losses import triplet_loss from utils import load_samples, enable_memory_growth, load_yaml print("Tensorflow Version : {}".format(tf.__version__)) enable_memory_growth() cfg = load_yaml("configs/facenet_triplet_mixed.yaml") # Load images dataset_folder = "datasets" images_path = os.path.join(dataset_folder, "ms1m_align_112/imgs_mixed/") X = load_samples(images_path, 0) counter = 0 for k, v in X.items(): counter += len(v) print("============================") print('{} Images loaded within {} classes'.format(counter, len(X.keys()))) print("============================") generator_config = { "IMAGE_W": cfg['input_size'], "IMAGE_H": cfg['input_size'], "IMAGE_C": 3, "BATCH_SIZE": cfg['batch_size'], "DATASET_PATH": images_path, "EMB_SIZE": cfg['embd_shape'], "CUT_FACE": cfg['is_ccrop'],
default='/home/mlpa/Workspace/dataset/etri_action_data/30_10/posetrack', help='base path of dataset.') parser.add_argument('--save_path', type=str, default='training_results/0000-00-00_00-00-00', help='model save path.') parser.add_argument('--save_latent', action='store_true', default=False, help='save latent variables') options = parser.parse_args() print(options) if __name__ == "__main__": test_info, test_data, _ = load_samples(options.data_path) model = load_model(options.model_path) predictions = model.predict(test_data) save_samples(os.path.join(options.save_path, 'recons'), predictions, test_info) #print(test_data.shape) save_recon_error(options.save_path, test_data, predictions, test_info) if options.save_latent: encoder = Model(inputs=model.get_layer('sequential_1').model.input, outputs=model.get_layer('sequential_1').model.output) latents = encoder.predict(test_data) save_latent_variables(os.path.join(options.save_path, 'latents'), latents, test_info)
except ValueError: print("Insert a number") continue else: break dataframe = pd.read_csv(CSV_PATH) target = int(target) print("\nStart of the attack\n") model = call_model.load_model() print("++++++++++ IN DISTRIBUTION ATTACK ++++++++++\n") x, y, masks = utils.load_samples(SAMPLE_IMG_DIR, SAMPLE_LABEL, target) y_target = np.zeros((len(x))) + target y_target = keras.utils.to_categorical(y_target, NUM_LABELS) y_true = np.zeros((len(x))) + y y_true = keras.utils.to_categorical(y_true, NUM_LABELS) utils.printProgressBar(0, 100, prefix='Progress FAST GRADIENT TARGET ATTACK:', suffix='Complete', length=50) x_fg_target = fg(model, x, y_target, masks, True) # FG TARGET ATTACK utils.printProgressBar(100, 100,