def init():
print 'Loading training samples..'
training_samples = utils.load_samples('../data/askubuntu/train_random.txt')
print len(training_samples)
print 'Loading dev samples..'
dev_samples = utils.load_samples('../data/askubuntu/dev.txt')
print len(dev_samples)
print 'Loading test samples..'
test_samples = utils.load_samples('../data/askubuntu/test.txt')
print len(test_samples)
print 'Loading corpus..'
question_map = utils.load_corpus('../data/askubuntu/text_tokenized.txt')
print len(question_map)
print 'Loading stop words..'
stop_words = utils.load_stop_words('../data/english_stop_words.txt')
print len(stop_words)
corpus_texts = map(lambda (t, b): t + ' ' + b, question_map.values())
print 'Loading embeddings..'
embedding_map = utils.load_embeddings(
'../data/pruned_askubuntu_android_vector.txt', corpus_texts,
stop_words)
print len(embedding_map)
print
utils.store_embedding_map(embedding_map)
return (training_samples, dev_samples, test_samples, question_map,
embedding_map)
def init():
print 'Loading askubuntu training samples..'
askubuntu_training_samples = utils.load_samples(
'../data/askubuntu/train_random.txt')
print len(askubuntu_training_samples)
print 'Loading askubuntu dev samples..'
askubuntu_dev_samples = utils.load_samples('../data/askubuntu/dev.txt')
print len(askubuntu_dev_samples)
print 'Loading askubuntu test samples..'
askubuntu_test_samples = utils.load_samples('../data/askubuntu/test.txt')
print len(askubuntu_test_samples)
print 'Loading askubuntu corpus..'
askubuntu_question_map = utils.load_corpus(
'../data/askubuntu/text_tokenized.txt')
print len(askubuntu_question_map)
print 'Loading android dev samples..'
android_dev_samples = utils.load_samples_stupid_format(
'../data/android/dev.pos.txt', '../data/android/dev.neg.txt')
print len(android_dev_samples)
print 'Loading android test samples..'
android_test_samples = utils.load_samples_stupid_format(
'../data/android/test.pos.txt', '../data/android/test.neg.txt')
print len(android_test_samples)
print 'Loading android corpus..'
android_question_map = utils.load_corpus('../data/android/corpus.tsv')
print len(android_question_map)
print 'Loading stop words..'
stop_words = utils.load_stop_words('../data/english_stop_words.txt')
print len(stop_words)
corpus_texts = map(lambda (t, b): t + ' ' + b,
askubuntu_question_map.values() + android_question_map.values())
print 'Loading embeddings..'
embedding_map = utils.load_embeddings(
'../data/pruned_android_vector.txt', corpus_texts, stop_words) # pruned_askubuntu_android_vector.txt
print len(embedding_map)
print
utils.store_embedding_map(embedding_map)
return (
askubuntu_training_samples,
askubuntu_dev_samples,
askubuntu_test_samples,
askubuntu_question_map,
android_dev_samples,
android_test_samples,
android_question_map,
embedding_map)
def validate_decoder(target_path, generation_path):
from numpy import linalg as LA
import numpy as np
target_info, targets, _ = load_samples(target_path)
generate_info, generates, _ = load_samples(generation_path)
if len(targets) != len(generates):
return False
for i in range(len(targets)):
if LA.norm(np.subtract(targets[i], generates[i])) > 0.001:
return False
return True
def main(args):
if args.get('verbose', False):
print(args)
recovery_method_name = args["recovery_method"]
recovery_params = args["recovery_params"]
RecoveryMethodClass = getattr(recovery, recovery_method_name)
graph = args.get("graph")
if graph is None:
graph = load_graph(args["graph_file"])
samples = args.get("samples")
if samples is None:
samples = load_samples(args["samples_file"])
recovery_method = RecoveryMethodClass(graph, samples, recovery_params)
x = [graph.node[idx]['value'] for idx in sorted(graph.node)]
x_hat = recovery_method.run()
results = args.copy()
results.update({"x_hat": x_hat, "nmse": nmse(x, x_hat)})
results_file = args.get("results_file")
if results_file is None:
return results
else:
dump_results(results, results_file)
def train_network(opts):
# load data
action_info, action_data, target_data = load_samples(
opts.data_path, options.dirty_sample_path)
# construct model
if 'AE' == opts.model:
model, encoder, decoder = ConvAE(opts.nfs, opts.sks, opts.nz,
opts.input_size, opts.dropout)
# elif 'VAE' == opts.model:
# model = ConvVAE(opts.nfs, opts.sks, opts.nz, opts.input_size)
# model.compile(optimizer='rmsprop', loss=vae_loss)
# model.fit(inputs=[action_data, target_data], epochs=opts.epochs, batch_size=opts.batch_size,
# callbacks=[tbCallBack, BestLossCallBack(model, opts.save_path, opts.save_period)], shuffle=True)
else:
print('There is no proper model for option: ' + opts.model)
return
callback_list = [
tbCallBack,
BestLossCallBack(model, opts.save_path, opts.save_period)
]
# BestLossCallBack(decoder, opts.save_path, opts.save_period, 'decoder')
# training
model.compile(optimizer='rmsprop', loss='mse')
model.fit(action_data,
target_data,
epochs=opts.epochs,
batch_size=opts.batch_size,
callbacks=callback_list,
shuffle=True)
def load_data(prob_folder, logfile):
files = list(pathlib.Path(prob_folder).glob('sample_*.pkl'))[:100]
xs, ys, cands = load_samples(files, logfile)
x_shift = xs.mean(axis=0)
x_scale = xs.std(axis=0)
x_scale[x_scale == 0] = 1
xs = (xs - x_shift) / x_scale
return xs, ys, cands
def instance(request):
if request.method == 'POST':
request_data = json.loads(request.body)
data_index = request_data['dataIndex']
data, _ = load_samples(dataset=rs['dataset'],
client_name=rs['client'],
sampling_type='local')
return JsonResponse({'data': data[data_index].tolist()})
def load(begin, to):
# Data loading
xs, ys, cands = load_samples(files[begin:to], logfile)
log(f" {xs.shape[0]} training samples", logfile)
# Data normalization
x_shift = xs.mean(axis=0)
x_scale = xs.std(axis=0)
x_scale[x_scale == 0] = 1
xs = (xs - x_shift) / x_scale
return xs, ys, cands
def attribute(request):
if request.method == 'POST':
request_data = json.loads(request.body)
dim_index = request_data['dimIndex']
data, _ = load_samples(dataset=rs['dataset'],
client_name=rs['client'],
sampling_type='local')
attr_data = data[:, dim_index]
return JsonResponse({'data': attr_data.tolist()})
def sampling(request):
if request.method == 'POST':
request_data = json.loads(request.body)
sampling_type = request_data['samplingType']
samples, ground_truth = load_samples(dataset=rs.state['dataset'],
client_name=rs.state['client'],
sampling_type=sampling_type)
rs.set('data', samples)
rs.set('sampling_type', sampling_type)
rs.set('ground_truth', ground_truth)
return JsonResponse({})
def cpca_cluster(request):
if request.method == 'POST':
request_data = json.loads(request.body)
alpha = None
if 'alpha' in request_data.keys():
alpha = request_data['alpha']
bg_data_idx = request_data['dataIndex']
data = rs.state['data']
homo_idx = rs.state['outputs_server'] == rs.state['outputs_client']
bg = data[bg_data_idx]
if bg.shape[0] == 1:
return JsonResponse(rs.state['cpca_all_result'])
fg = np.concatenate((data[homo_idx], bg))
local_data, _ = load_samples(dataset=rs['dataset'],
client_name=rs['client'],
sampling_type='local')
if USE_GPU:
with torch.no_grad():
cPCA = CPCA_GPU(n_components=2)
cPCA.fit(target=fg, background=bg, alpha=alpha)
projected_data = cPCA.transform(local_data)
components = cPCA.get_components()
alpha = cPCA.get_alpha()
else:
cPCA = CPCA(n_components=2)
cPCA.fit(target=fg, background=bg, alpha=alpha)
projected_data = cPCA.transform(local_data)
components = cPCA.get_components()
alpha = cPCA.get_alpha()
projected_data = np.round(projected_data, 6).astype(float)
components = np.round(components, 6).astype(float)
alpha = np.round(alpha, 6).astype(float)
data = {
'alpha': alpha,
'cPC1': components[0].tolist(),
'cPC2': components[1].tolist(),
'projectedData': projected_data.tolist(),
}
return JsonResponse(data)
def compute_cov(state):
"""Computes the crosspectrum matrices per subjects."""
for sub in SUBJECT_LIST:
pattern = prefix + "_s{}_{}.mat"
file_path = path(SAVE_PATH / pattern.format(sub, state))
if not file_path.isfile():
# data must be of shape n_trials x n_elec x n_samples
data = load_samples(DATA_PATH, sub, state)
if FULL_TRIAL:
data = np.concatenate(data, axis=1)
data = data.reshape(1, data.shape[0], data.shape[1])
cov = Covariances()
mat = cov.fit_transform(data)
savemat(file_path, {"data": mat})
def __init__(self, graph, samples):
super().__init__()
if isinstance(graph, nx.Graph):
self.graph = graph
elif isinstance(graph, str):
self.graph = load_graph(graph)
else:
raise ValueError("unexpected graph type: {}".format(type(graph)))
if isinstance(samples, (list, tuple, set)):
self.samples = samples
elif isinstance(samples, str):
self.samples = load_samples(samples)
else:
raise ValueError("unexpected samples type: {}".format(type(samples)))
def computeAndSavePSD(
SUBJECT_LIST, state, freq, window, overlap, fmin, fmax, fs, elec=None
):
"""loads data, compute PSD and saves PSD of all subjects in one file"""
N_ELEC = 19 if elec is None else len(elec)
print(state, freq, "bande {}: [{}-{}]Hz".format(freq, fmin, fmax))
for elec in range(N_ELEC): # pour chaque elec
channel_name = CHANNEL_NAMES[elec]
file_path = path(
SAVE_PATH
/ "PSD_{}_{}_{}_{}_{:.2f}.mat".format(
# 'PSD_EOG_sleepState_%s_%s_%i_%i_%.2f.mat' %
state,
freq,
channel_name,
window,
overlap,
)
)
if not file_path.isfile():
psds = []
for sub in SUBJECT_LIST: # pour chaque sujet
X = load_samples(DATA_PATH, sub, state)
psd_list = []
for j in range(X.shape[0]): # pour chaque trial
psd = computePSD(
X[j, elec],
window=window,
overlap=OVERLAP,
fmin=fmin,
fmax=fmax,
fs=fs,
)
psd_list.append(psd)
psd_list = np.asarray(psd_list)
psds.append(psd_list.ravel())
print(file_path)
savemat(file_path, {"data": psds})
def main(argv):
# creating the algorithms. the optimal hyper-parameters are hardcoded.
per = Perceptron()
svm = SVM()
pa = PA()
# loading the train set, including the labels on the rightmost column.
train_set = utils.load_dset(argv[0], argv[1])
# loading the test set. add a right column of ones for the bias
test_x = utils.load_samples(argv[2])
# normalizing the train and the test sets with min-max norm. min = 4, max = 30.
mins, denoms = norm.minmax_params(train_set)
train_set = norm.minmax(train_set, 4, 30)
test_x = norm.minmax(test_x, 4, 30, mins, denoms)
# training the algorithms .
train(train_set, per, svm, pa, 100)
# predicting and printing the results on the test set.
predict(test_x, per, svm, pa)
def compute_cosp(state, freq, window, overlap, cycle=None):
"""Computes the crosspectrum matrices per subjects."""
if cycle is not None:
print(state, freq, cycle)
else:
print(state, freq)
freqs = FREQ_DICT[freq]
for sub in SUBJECT_LIST:
if cycle is None:
file_path = SAVE_PATH / prefix + "_s{}_{}_{}_{}_{:.2f}.mat".format(
sub, state, freq, window, overlap)
else:
file_path = SAVE_PATH / prefix + "_s{}_{}_cycle{}_{}_{}_{:.2f}.mat".format(
sub, state, cycle, freq, window, overlap)
if not file_path.isfile():
# data must be of shape n_trials x n_elec x n_samples
if cycle is not None:
data = load_full_sleep(DATA_PATH, sub, state, cycle)
if data is None:
continue
data = data.swapaxes(1, 2)
else:
data = load_samples(DATA_PATH, sub, state)
if FULL_TRIAL:
data = np.concatenate(data, axis=1)
data = data.reshape(1, data.shape[0], data.shape[1])
cov = CospCovariances(window=window,
overlap=overlap,
fmin=freqs[0],
fmax=freqs[1],
fs=SF)
mat = cov.fit_transform(data)
if len(mat.shape) > 3:
mat = np.mean(mat, axis=-1)
savemat(file_path, {"data": mat})
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
from keras.models import Sequential
from keras.layers import Lambda, Flatten, Dense, Cropping2D, Convolution2D, MaxPooling2D, Dropout
from keras.optimizers import Adam
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint, Callback
from keras.utils.visualize_util import plot
import utils
import constants as CONST
# LOAD SAMPLES:
# Load all samples from one or multiple files.
all_samples = utils.load_samples(CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE,
CONST.BEACH_DATA_FILES)
# TRAIN/VALIDATION SPLITS:
# Split all the samples in training and validation and calculate the number of samples in each set taking into account
# the criteria and methods that will be used to augment them.
train_samples, validation_samples = train_test_split(all_samples,
test_size=CONST.TEST_SIZE)
TRAIN_SAMPLES_COUNT = utils.calculate_augmented_size(train_samples,
CONST.ANGLE_CORRECTION,
CONST.SKIP_FILTER)
VALIDATION_SAMPLES_COUNT = utils.calculate_augmented_size(
validation_samples, CONST.ANGLE_CORRECTION, CONST.SKIP_FILTER)
# IMAGE GENERATOR:
from sklearn.model_selection import train_test_split from sklearn.linear_model import SGDClassifier from utils import load_samples, count_files, evaluate, imagenet_preprocess_input, _set_n_retrain, _get_weighted_layers, producer, getPaths, load_data, load_data_part, categorical_acc, load_data_frac from networks import simple_gram, resnet_trained #Must modify the paths so that they point to your local wikipaintings files data = "wikiart/wikiart/" data_train = r"C:\Users\tgill\OneDrive\Documents\GD_AI\ArtGAN\wikipaintings_full\wikipaintings_train" data_test = r"C:\Users\tgill\OneDrive\Documents\GD_AI\ArtGAN\wikipaintings_full\wikipaintings_val" target_size = (224, 224) X, y, classes = load_samples(10) X_test, y_test, classes_test = load_samples(10, data_test) #m = resnet_trained(20) m = simple_gram(20) print(m.summary()) n_files_train = count_files(data_train) n_files_test = count_files(data_test) nepochs = 20 epoch_size = 2500 batch_size = 32 steps_per_epoch = (n_files_train//batch_size) v_step = 50 #n_files_test//batch_size
import matplotlib.pyplot as plt
import math
from sklearn.model_selection import train_test_split
import utils
import constants as CONST
# LOAD DATA:
# Track 1 (Beach):
samples_beach = utils.load_samples(
CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE, [
CONST.BEACH_4_ANTICLOCK_FILE, CONST.BEACH_4_CLOCK_FILE,
CONST.RECO_BEACH_1_ANTICLOCK_FILE, CONST.RECO_BEACH_1_CLOCK_FILE
])
# Track 2 (Mountain):
samples_mountain = utils.load_samples(
CONST.DATA_DIR, CONST.IMG_DIR, CONST.LOG_FILE, [
CONST.MOUNTAIN_4_ANTICLOCK_FILE, CONST.MOUNTAIN_4_CLOCK_FILE,
CONST.RECO_MOUNTAIN_1_ANTICLOCK_FILE, CONST.RECO_MOUNTAIN_1_CLOCK_FILE
])
# EXTRACT BASE ANGLES:
angles_beach = list(map(lambda sample: sample["steering"], samples_beach))
angles_mountain = list(map(lambda sample: sample["steering"],
samples_mountain))
import numpy as np
import progressbar
import os
import re
import glob
from utils import load_samples, save_samples
kDataRoot = "C:/Users/JM/Desktop/Data/ETRIrelated/BMVC"
kActionRoot = os.path.join(kDataRoot, "etri_action_data_neck_point_0_0")
if __name__ == "__main__":
all_data, _, file_names = load_samples(kActionRoot)
f1 = 2 * precision * recall / (precision + recall)
f1s_list.append(f1)
except ZeroDivisionError:
f1s_list.append(np.nan)
return list(mse_grid)[int(np.nanargmax(f1s_list))]
if __name__ == "__main__":
for k in range(10):
# validation step for find threshold value
video_list = cv_file_list(k, 'validation') # k = 1
data_info, action_data, _ = load_samples(kActionData)
val_data = []
val_info = []
for i, info in enumerate(data_info):
video_name = str(int(info['video_name']))
if video_name in video_list:
val_data.append(action_data[i])
val_info.append(info)
val_data = np.array(val_data)
# TODO: run model for make validation error
model_name = os.path.join(kDetectBase, "best_loss.hdf5")
model = load_model(model_name)
predictions = model.predict(val_data)
save_recon_error(kDetectBase, val_data, predictions, val_info,
"recon_val_error.csv")
from discriminator import Discriminator
from generator import Generator
from parameters import d_conv_dim, z_size, g_conv_dim, beta1, beta2, lr_d, lr_g
from train import train
from utils import display_images, weights_init_normal, gpu_check, load_samples, view_samples
display_images(loader)
D = Discriminator(d_conv_dim)
G = Generator(z_size=z_size, conv_dim=g_conv_dim)
D.apply(weights_init_normal)
G.apply(weights_init_normal)
train_on_gpu = gpu_check()
if not train_on_gpu:
print('No GPU found. Please use a GPU to train your neural network.')
else:
print('Training on GPU!')
d_optimizer = optim.Adam(D.parameters(), lr_d, [beta1, beta2])
g_optimizer = optim.Adam(G.parameters(), lr_g, [beta1, beta2])
n_epochs = 30
losses = train(D, d_optimizer, G, g_optimizer, n_epochs=n_epochs)
samples = load_samples()
_ = view_samples(-1, samples)
from batch_generator import BatchGenerator
from callbacks import SaveEmbeddingModel
from losses import triplet_loss
from utils import load_samples, enable_memory_growth, load_yaml
print("Tensorflow Version : {}".format(tf.__version__))
enable_memory_growth()
cfg = load_yaml("configs/facenet_triplet_mixed.yaml")
# Load images
dataset_folder = "datasets"
images_path = os.path.join(dataset_folder, "ms1m_align_112/imgs_mixed/")
X = load_samples(images_path, 0)
counter = 0
for k, v in X.items():
counter += len(v)
print("============================")
print('{} Images loaded within {} classes'.format(counter, len(X.keys())))
print("============================")
generator_config = {
"IMAGE_W": cfg['input_size'],
"IMAGE_H": cfg['input_size'],
"IMAGE_C": 3,
"BATCH_SIZE": cfg['batch_size'],
"DATASET_PATH": images_path,
"EMB_SIZE": cfg['embd_shape'],
"CUT_FACE": cfg['is_ccrop'],
default='/home/mlpa/Workspace/dataset/etri_action_data/30_10/posetrack',
help='base path of dataset.')
parser.add_argument('--save_path',
type=str,
default='training_results/0000-00-00_00-00-00',
help='model save path.')
parser.add_argument('--save_latent',
action='store_true',
default=False,
help='save latent variables')
options = parser.parse_args()
print(options)
if __name__ == "__main__":
test_info, test_data, _ = load_samples(options.data_path)
model = load_model(options.model_path)
predictions = model.predict(test_data)
save_samples(os.path.join(options.save_path, 'recons'), predictions,
test_info)
#print(test_data.shape)
save_recon_error(options.save_path, test_data, predictions, test_info)
if options.save_latent:
encoder = Model(inputs=model.get_layer('sequential_1').model.input,
outputs=model.get_layer('sequential_1').model.output)
latents = encoder.predict(test_data)
save_latent_variables(os.path.join(options.save_path, 'latents'),
latents, test_info)
except ValueError:
print("Insert a number")
continue
else:
break
dataframe = pd.read_csv(CSV_PATH)
target = int(target)
print("\nStart of the attack\n")
model = call_model.load_model()
print("++++++++++ IN DISTRIBUTION ATTACK ++++++++++\n")
x, y, masks = utils.load_samples(SAMPLE_IMG_DIR, SAMPLE_LABEL, target)
y_target = np.zeros((len(x))) + target
y_target = keras.utils.to_categorical(y_target, NUM_LABELS)
y_true = np.zeros((len(x))) + y
y_true = keras.utils.to_categorical(y_true, NUM_LABELS)
utils.printProgressBar(0,
100,
prefix='Progress FAST GRADIENT TARGET ATTACK:',
suffix='Complete',
length=50)
x_fg_target = fg(model, x, y_target, masks, True) # FG TARGET ATTACK
utils.printProgressBar(100,
100,