def main(opts):
dataset = opts.dataset
embed_dim = int(opts.dimension)
# File that contains the edges. Format: source target
# Optionally, you can add weights as third column: source target weight
edge_f = 'Data/%s.edgelist' % dataset
# Specify whether the edges are directed
# isDirected = True
print "Loading Dataset"
# Load graph
G = graph_util.loadGraphFromEdgeListTxt(edge_f, directed=False)
#G = G.to_directed()
embedding = LaplacianEigenmaps(d=embed_dim)
print('Num nodes: %d, num edges: %d' %
(G.number_of_nodes(), G.number_of_edges()))
t1 = time()
# Learn embedding - accepts a networkx graph or file with edge list
print "Starting Embedding"
Y, t = embedding.learn_embedding(graph=G,
edge_f=None,
is_weighted=True,
no_python=True)
print(embedding._method_name + ':\n\tTraining time: %f' % (time() - t1))
np_save(writable("Embedding_Results", "jac_" + dataset + str(embed_dim)),
Y)
def array2data(arr, adress, other='C', delimiter='\t'):
'''Save an array (max 2D for dat/txt) as "*.dat/txt" or "*.npy".'''
if adress.endswith('.npy'):
np_save(adress, arr)
elif adress.endswith('.txt')\
or adress.endswith('.asc')\
or adress.endswith('.dat'):
ofi = open(adress, 'w')
try:
for i in xrange(arr.shape[0]):
for j in xrange(arr.shape[1]):
ofi.write(str(arr[i, j]) + delimiter)
ofi.write('\n')
except:
for i in xrange(arr.shape[0]):
ofi.write(str(arr[i]) + delimiter)
ofi.close()
elif adress.endswith('.mat'):
from scipy.io import savemat
if type(arr) == dict:
savemat(adress, arr)
else:
savemat(adress, {other: arr})
def main(opts):
dataset = opts.dataset
embed_dim = int(opts.dimension)
# File that contains the edges. Format: source target
# Optionally, you can add weights as third column: source target weight
edge_f = 'Data/%s.edgelist' % dataset
G = nx.read_edgelist(edge_f)
A = nx.adjacency_matrix(G)
num_points = A.shape[0]
D = np.sum(A, axis=0)
D = np.squeeze(np.asarray(D))
A = csr_matrix(A)
res = optimize.minimize(partial(num_lap,
A=A,
n=num_points,
dim=embed_dim,
D=D,
c=1),
np.random.rand(num_points * embed_dim),
jac=partial(lap_jac,
A=A,
n=num_points,
dim=embed_dim,
c=1))
print(res.fun)
t1 = time()
Y = res.x.reshape(num_points, embed_dim)
print(embedding._method_name + ':\n\tTraining time: %f' % (time() - t1))
np_save(
writable("Embedding_Results", dataset + str(embed_dim) + opts.cost), Y)
def save(self):
if not exists(self.directory):
mkdir(self.directory)
arr = [self.num_steps, self.action_num]
np_save(file=join(self.directory, self.name), arr=arr)
np_save(file=join(self.directory, self.name + "_eps"), arr=self.eps)
def save(self):
from numpy import save as np_save
from os import mkdir
from os.path import exists, join
if not exists(self.__directory):
mkdir(self.__directory)
for changed in self.__change_log:
path = join(self.__directory, f"memory_{changed:05d}")
np_save(file=path, arr=self.__episodes[changed])
self.__change_log.clear()
path = join(self.__directory, f"memory_meta")
np_save(file=path, arr=[self.__pos, self.__capacity])
def log_transitions(self):
if len(self.memory) > 0:
basename = self.logdir + "/{}.{}".format(
self.environment_name,
datetime.now().strftime("%Y-%m-%d-%H-%M-%s"))
print("Base Filename: ", basename, flush=True)
state, action, reward, next_state, done = zip(*self.memory)
np_save(basename + "-state.npy",
np_array(state),
allow_pickle=False)
np_save(basename + "-action.npy",
np_array(action),
allow_pickle=False)
np_save(basename + "-reward.npy",
np_array(reward),
allow_pickle=False)
np_save(basename + "-nextstate.npy",
np_array(next_state),
allow_pickle=False)
np_save(basename + "-done.npy", np_array(done), allow_pickle=False)
self.memory.clear()
def learn_manifold(self, X, manifold_out_file_name=None):
self.debug_string_out.clear()
self.print_and_remember("Learning manifold(" + self.manifold_learner +
")" + str(datetime.now()))
learn_time = time()
if manifold_out_file_name is not None and isfile(
manifold_out_file_name
): # check the learned manifold existance
manifold_feats = np_load(manifold_out_file_name, allow_pickle=True)
self.print_and_remember("Manifold loaded(" +
manifold_out_file_name + ")")
elif self.manifold_learner == 'UMAP':
manifold_feats = UMAP(random_state=0,
metric=self.dist_metric,
n_components=self.manifold_dimension,
n_neighbors=self.num_of_neighbours,
min_dist=float(
self.min_dist)).fit_transform(X)
elif self.manifold_learner == 'LLE':
manifold_feats = LocallyLinearEmbedding(
n_components=self.manifold_dimension,
n_neighbors=self.num_of_neighbours).fit_transform(X)
elif self.manifold_learner == 'tSNE':
manifold_feats = TSNE(n_components=self.manifold_dimension,
random_state=0,
verbose=0).fit_transform(X)
elif self.manifold_learner == 'isomap':
manifold_feats = Isomap(
n_components=self.manifold_dimension,
n_neighbors=self.num_of_neighbours).fit_transform(X)
self.print_and_remember(
"Time to learn manifold: " +
str(funcH.getElapsedTimeFormatted(time() - learn_time)))
if manifold_out_file_name is not None:
np_save(manifold_out_file_name, manifold_feats, allow_pickle=True)
self.print_and_remember("Manifold saved(" +
manifold_out_file_name + ")")
return manifold_feats, self.debug_string_out
def __call__(self,
img_processed,
img_normalized,
img_with_background=None):
if not Capture.capture:
return
Capture.capture = False
# When files are listed they will be sorted by subject, then capture number, then capture type
filename_basis = GlobalConfig.Paths.data_dir + Capture.subject + '_' + str(
Capture.subject_counter).zfill(4)
np_save(filename_basis + '_depth.npy', img_processed)
cv2.imwrite(filename_basis + '_depth_normalized.png', img_normalized)
if not img_with_background is None:
np_save(filename_basis + '_depth_withbg.npy', img_with_background)
img_with_background = NormalizeFor8Bit(img_with_background)
cv2.imwrite(filename_basis + '_depth_withbg.png',
img_with_background)
Capture.subject_counter += 1
print('Captured "' + filename_basis + '"')
def compute_tsne_positions(activations_per_point: array, perplexity: int, name: str) -> array:
"""
Computes t-SNE positions from a dataset.
`activations_per_point`: n_obvs x n_dims
"""
logger.info(f"TSNE from data of size {activations_per_point.shape}")
t_sne_positions_path = Path(TSNE_SAVE_DIR, f"t-sne positions {name} perp={perplexity}.npy")
if t_sne_positions_path.exists():
logger.info("Loading...")
t_sne_positions = np_load(t_sne_positions_path)
else:
logger.info("Computing...")
t_sne_positions = TSNE(
n_components=2, # 2D
perplexity=perplexity,
# Recommended args
n_iter=1_000,
learning_rate=200,
method="barnes_hut",
).fit_transform(activations_per_point)
np_save(t_sne_positions_path, t_sne_positions)
return t_sne_positions
probs = mdl.predict_proba(X_test)
y_pred = probs[:,1]
cv_scores[c] = qual(y_test, y_pred)
tech_data.append(cv_scores.copy())
output.write('%s,%d,%f,%f,%f,%f,%s\n' %
('Guess', d, np.mean(cv_scores), np.std(cv_scores), t, p, args.label))
plot_data.append(tech_data)
output.close()
names = techniques + ["None", "Guessing"]
points = [str(step) for step in dim_steps]
# Save off plotting data
np_save(path.join(cv_dir, 'plot_data.npy'), array(plot_data))
# Save off additional data
output = open(path.join(cv_dir, 'plot_data.txt'), 'w')
output.write('#TECHNIQUES,' + ','.join(names) + '\n')
output.write('#STEPS,' + ','.join(points) + '\n')
output.close()
fig, ax = plt.subplots(figsize=(10, 8), dpi=80)
plot_data = array(plot_data)
lgd = make_grouped_box(ax, plot_data, names, xticklabels=points, legend_pos='outside')
ax.set_ylabel('AUC')
ax.set_xlabel('Number of Dimensions')
ax.set_title('Predicting "%s"' % (metadata_category))
plt.savefig(args.output_file, bbox_extra_artists=(lgd,), bbox_inches='tight')
def test(primaryDataSet, helperDataSet, primaryClass, helperClass, primaryInstances, helperInstances):
'''
Inputs :
dataSets : List : Datasets for which samples are to be genrated
instances : List : Number of instances to be used from original dataset
classes : List : Classes for which samples are to be generated
Output :
File with 1000 compressed images generated by GAN
'''
helperClass = primaryClass
modelFolder = resultDir + 'models/crossDataSetMMDall'+'/'+primaryDataSet+'/'
print (primaryDataSet, helperDataSet, primaryClass, helperClass, primaryInstances, helperInstances, getEpochs(primaryDataSet,primaryInstances)-1)
modelFile = modelFolder + primaryDataSet + '_' + helperDataSet + '_' + \
str(primaryClass) + '_' + str(helperClass) + '_' + \
str(primaryInstances) + '_' + str(helperInstances)+'_'+ \
str(getEpochs(primaryDataSet,primaryInstances)-1)+'.pt'
print ('Generating examples for Dataset: '+primaryDataSet+
' Primary Class: '+str(primaryClass)+
' Helper Class: '+str(helperClass)+
' Primary Instances: '+str(primaryInstances)+
' Helper Instances: '+str(helperInstances)+
' Epochs: '+str(getEpochs(primaryDataSet,primaryInstances)))
numOutputChannels = getChannels(primaryDataSet)
# load the model learnt during training
G = Generator(numInputChannels, numGenFilter, numOutputChannels)
G.load_state_dict(torch.load(modelFile))
genImageConcat = np.empty(1)
iterations = numOfSamples/batchSize
for iteration in range(iterations):
noise = torch.FloatTensor(batchSize,
numInputChannels,
1,
1)
noise.normal_(0,1)
if cuda:
G = G.cuda()
noise = noise.cuda()
noiseVariable = Variable(noise)
genImage = G(noiseVariable)
genImage = genImage.data
genImage = genImage.cpu()
genImage = genImage.numpy()
if iteration==0:
genImageConcat = genImage
else:
genImageConcat = np.concatenate((genImageConcat, genImage),
axis=0)
if iteration==(iterations-1):
# normalize sets image pixels between 0 and 1
genImage = torchvision.utils.make_grid(torch.from_numpy(genImage[:25]), nrow=5, normalize=True)
# mapping between 0 to 1 as required by imshow,
# otherwise the images are stored in the form -1 to 1
# done through normalize=True
genImage = genImage.permute(1,2,0)
genImage = genImage.numpy()
plt.imshow(genImage, cmap='gray')
plotFolder = resultDir+'results'+'/'+'crossDataSetMMDall/samples'+'/'+primaryDataSet+'/'
checkAndCreateFolder(plotFolder)
plotFile = primaryDataSet + '_' + helperDataSet + '_' + \
str(primaryClass) + '_' + 'all' + '_' + \
str(primaryInstances) + '_' + str(helperInstances)
plotPath = plotFolder + plotFile
plt.axis('off')
plt.savefig(plotPath, bbox_inches='tight')
plt.show()
resultFolder = resultDir+'results/crossDataSetMMDall'+'/'+'compressed'+'/'+primaryDataSet+'/'
checkAndCreateFolder(resultFolder)
resultFile = primaryDataSet + '_' + helperDataSet + '_' + \
str(primaryClass) + '_' + 'all' + '_' + \
str(primaryInstances) + '_' + str(helperInstances) + '.npy'
resultPath = resultFolder + resultFile
# save the image in some format
with open(resultPath,'wb+') as fh:
genImageConcat = np.squeeze(genImageConcat)
np_save(fh, genImageConcat, allow_pickle=False)
sync(fh)
def fit_ptsne_model(model: torch.nn.Module, input_points: Dataset,
opt: Optimizer, perplexity: Optional[int], n_epochs: int,
dev: str, save_dir_path: str,
epochs_to_save_after: Optional[int],
early_exaggeration: int, early_exaggeration_constant: int,
batch_size: int, dist_func_name: str,
bin_search_tol: float, bin_search_max_iter: int,
min_allowed_sig_sq: float, max_allowed_sig_sq: float,
configuration_report: str) -> None:
"""
Fits a parametric t-SNE model and optionally saves it to the desired directory.
Fits either regular or multi-scale t-SNE
:param model: nn.Module instance
:param input_points: tensor of original points
:param opt: optimizer instance
:param perplexity: perplexity of a model. If passed None, multi-scale parametric t-SNE
model will be trained
:param n_epochs: Number of epochs for training
:param dev: device for tensors (e.g. "cpu" or "cuda")
:param save_dir_path: path to directory to save a trained model to
:param epochs_to_save_after: number of epochs to save a model after. If passed None,
model won't be saved at all
:param early_exaggeration: Number of first training cycles in which
exaggeration will be applied
:param early_exaggeration_constant: Constant by which p_joint is multiplied in early exaggeration
:param batch_size: Batch size for training
:param dist_func_name: Name of distance function for distance matrix.
Possible names: "euc", "jaccard", "cosine"
:param bin_search_tol: A small number - tolerance threshold for binary search
:param bin_search_max_iter: Number of max iterations for binary search
:param min_allowed_sig_sq: Minimum allowed value for the spread of any distribution
in conditional probability matrix
:param max_allowed_sig_sq: Maximum allowed value for the spread of any distribution
in conditional probability matrix
:param configuration_report: Config of the model in string form for report purposes
:return:
"""
model.train()
batches_passed = 0
model_name = get_random_string(6)
epoch_losses = []
# Function operates with DataLoader
train_dl = DataLoader(input_points, batch_size=batch_size, shuffle=True)
for epoch in range(n_epochs):
train_loss = 0
epoch_start_time = datetime.datetime.now()
# For every batch
for list_with_batch in tqdm(train_dl):
orig_points_batch, _ = list_with_batch
# Calculate conditional probability matrix in higher-dimensional space for the batch
# Regular parametric t-SNE
if perplexity is not None:
target_entropy = log2(perplexity)
p_cond_in_batch = calculate_optimized_p_cond(
orig_points_batch, target_entropy, dist_func_name,
bin_search_tol, bin_search_max_iter, min_allowed_sig_sq,
max_allowed_sig_sq, dev)
if p_cond_in_batch is None:
continue
p_joint_in_batch = make_joint(p_cond_in_batch)
# Multiscale parametric t-SNE
else:
max_entropy = round(log2(batch_size / 2))
n_different_entropies = 0
mscl_p_joint_in_batch = zeros(batch_size,
batch_size).to(device(dev))
for h in range(1, max_entropy):
p_cond_for_h = calculate_optimized_p_cond(
orig_points_batch, h, dist_func_name, bin_search_tol,
bin_search_max_iter, min_allowed_sig_sq,
max_allowed_sig_sq, dev)
if p_cond_for_h is None:
continue
n_different_entropies += 1
p_joint_for_h = make_joint(p_cond_for_h)
# TODO This fails if the last batch doesn't match the shape of mscl_p_joint_in_batch
mscl_p_joint_in_batch += p_joint_for_h
p_joint_in_batch = mscl_p_joint_in_batch / n_different_entropies
# Apply early exaggeration to the conditional probability matrix
if early_exaggeration:
p_joint_in_batch *= early_exaggeration_constant
early_exaggeration -= 1
batches_passed += 1
opt.zero_grad()
# Calculate joint probability matrix in lower-dimensional space for the batch
embeddings = model(orig_points_batch)
q_joint_in_batch = get_q_joint(embeddings, "euc", alpha=1)
# Calculate loss
loss = loss_function(p_joint_in_batch, q_joint_in_batch)
train_loss += loss.item()
# Make an optimization step
loss.backward()
opt.step()
epoch_end_time = datetime.datetime.now()
time_elapsed = epoch_end_time - epoch_start_time
# Report loss for epoch
average_loss = train_loss / batches_passed
epoch_losses.append(average_loss)
print(
f'====> Epoch: {epoch + 1}. Time {time_elapsed}. Average loss: {average_loss:.4f}',
flush=True)
# Save model and loss history if needed
save_path = os.path.join(save_dir_path,
f"{model_name}_epoch_{epoch + 1}")
if epochs_to_save_after is not None and (
epoch + 1) % epochs_to_save_after == 0:
torch.save(model, save_path + ".pt")
with open(save_path + ".json", "w") as here:
json.dump(json.loads(configuration_report), here)
print('Model saved as %s' % save_path, flush=True)
if epochs_to_save_after is not None and epoch == n_epochs - 1:
epoch_losses = array(epoch_losses)
loss_save_path = save_path + "_loss.npy"
np_save(loss_save_path, epoch_losses)
print("Loss history saved in", loss_save_path, flush=True)
NormW = 4
dir_pri1 = './AT_tT1t_X_Patt'
dir_pri2 = '../FilesIN/Time_Courses/only_positives'
dir_pri3 = './fRMI_only_positives'
if not os.path.exists(dir_pri1):
print 'Pas de dossier ' + dir_pri1
quit()
if not os.path.exists(dir_pri2):
print 'Pas de dossier ' + dir_pri2
quit()
if not os.path.exists(dir_pri3):
os.mkdir(dir_pri3)
t0 = time()
pool = multiprocessing.Pool()
args = arange(0.02,0.981,0.03).tolist()
resultslist = pool.map(whatyouwant, args)
resultsarra = resultslist[0]
for ll in range(1, len(resultslist)):
resultsarra = append(resultsarra, resultslist[ll], axis=0)
np_save(dir_pri3 + '/correlations_W%.2f_p%i.npy' %(NormW, patient), resultsarra)
print "Simulation took :",time()-t0,"s"
print "min= %.2f ; max= %.2f" %(resultsarra.min(), resultsarra.max())
imshow(abs(resultsarra), cmap=get_cmap('Blues'), interpolation='nearest')
show()
else:
da = ncout.variables['delta_params5'][:]
ncout.close()
if FLAG_mns:
filestr_draws = "draws_" + str(ch) + "_" + str(npop) + ".npy"
filestr_ensemble = "ensemble_" + str(ch) + "_" + str(npop) + ".npy"
# Load / Generate draws
if FLAG_load_draws:
draws = np_load(filestr_draws)
else:
draws = ensemble_func.cov2draws(har, npop)
np_save(filestr_draws, draws, allow_pickle=False)
# Load / Generate ensemble
if FLAG_load_ensemble:
ens = np_load(filestr_ensemble, allow_pickle=True).item()
else:
ens = ensemble_func.draws2ensemble(draws, nens)
np_save(filestr_ensemble, ens, allow_pickle=True)
ensemble = ens['ensemble']
ensemble_idx = ens['ensemble_idx']
decile = ens['decile']
Z = ens['Z']
Z_norm = ens['Z_norm']
da = ensemble - a_ave # [nens,npar]
def file_np_save(path: str, np_array):
Path(path).parent.mkdir(parents=True, exist_ok=True)
np_save(path, np_array)
'#################### TC_998'
t0 = time()
for i in range(nTC):
TC1[i * step:(i + 1) * step, :] = np_load(dir_TC1 + dir_sub +
'TC_%i.npy' % i)
print 'TC 998 loaded : %.2fs' % (time() - t0)
'#################### BO_998'
t0 = time()
for k in range(N1):
Bk = convolve(TC1[:, k], Hrv)[:tmax]
for o in range(avgOn):
BO1[:, k] += Bk[o::avgOn]
BO1 /= avgOn
np_save(dir_BO1 + 'BO_%i_' % N1 + name + '.npy', BO1)
print 'BO 998 generated : %.2fs' % (time() - t0)
'#################### BO_66, FC_B66, FC_B998'
t0 = time()
for n in range(N1):
BO2[:, D998_D66[n]] += BO1[:, n]
for t in range(BO1.shape[0]):
BO2[t, :] /= AvgN_D66
np_save(dir_BO2 + 'BO_%i_' % N2 + name + '.npy', BO2)
FC2 = fastPearsonCorrelation(BO2)
del BO2
np_save(dir_FC2 + 'FC_B%i_' % N2 + name + '.npy', FC2)
del FC2
FC1 = fastPearsonCorrelation(BO1)
del BO1
def main():
x_data, y_data = load_data()
## Set parameter to True for initial download.
## Once data is present, set this to False to
## prevent re-downloading data.
## Plotting the Iris Petal and Sepal length and width.
plot_iris_data(x_data, y_data)
y_data = one_hot(y_data)
#Split data: 80% test set, 20% validation set.
i_80 = int(len(y_data) * 0.8)
x_train, y_train = x_data[:i_80], y_data[:i_80]
x_test, y_test = x_data[i_80:], y_data[i_80:]
iris_nn = NN_Model(
x_train, ## input data.
y_train, ## output data.
3, ## 3 NN layers: Input, hidden-layer, output.
[4, 4, 3])
## num of nodes for each layer.
if Grad_Descent_Method:
print("\nNeural Network XNOR - using GRADIENT DESCENT ITERATION\n",
"#" * 30, "\n")
# File location where learned weight is saved.
theta_file = CURRENT_PATH + r'/' + 'theta.npy'
if LOAD_PREV_THETAS:
flat_thetas = np_load(theta_file)
iris_nn.unflatten_Thetas(flat_thetas)
if CONTINUOUS_TRAINING:
iris_nn.train_NN()
np_save(theta_file, iris_nn.flatten_Thetas())
else:
iris_nn.train_NN()
np_save(theta_file, iris_nn.flatten_Thetas())
# Display final cost after learning iterations.
print("Final Cost J = ", iris_nn.J_cost(iris_nn.a[-1]))
if PLOT_COST:
# Plot the J Cost vs. # of iterations. J should coverge as iteration increases.
x_axis = range(len(iris_nn.J_cost_values))
y_axis = iris_nn.J_cost_values
plt.plot(x_axis, y_axis, label='J_cost vs. # of Iterations')
plt.show()
# Test model accuracy on Validation/Test set.
acc_count = 0
for i in range(len(x_test)):
x_input = x_test[i].flatten()
y_val = np.argmax(y_test[i])
y_pred = iris_nn.predict(x_input)[0]
#print(y_pred, y_val);
if y_pred == y_val: acc_count += 1
print("Test Accuraccy = {}".format(acc_count / len(x_test)))
return 0
def plotAccuracyBar(dataSet,
classifier,
primaryInstances,
helperInstances,
accuracyArray,
showImage = 1):
fig = plt.figure()
ax = fig.add_subplot(111)
ind = np.arange(len(primaryInstances))
width = 0.15
delta = 0.02
color = ['black', 'red', 'blue', 'green']
histList = []
legendList = []
for i in range(accuracyArray.shape[0]):
hist = ax.bar(ind+i*(width+delta), list(accuracyArray[:,i]), width, color=color[i])
histList.append(hist)
if i==0:
legendList.append('Original')
elif i==1:
legendList.append('GAN - 0 Helper Instances')
else:
legendList.append('GAN - '+str(helperInstances[i-2])+' Helper Instances')
print (accuracyArray)
ax.set_xlim(-4*width,len(ind)+width)
ax.set_ylim(0,1.5)
ax.set_ylabel('Accuracy [out of 1]')
ax.set_xlabel('Number of Primary Instances')
ax.set_title(dataSet)
ax.set_xticks(ind+width)
xtickNames = ax.set_xticklabels(primaryInstances)
plt.setp(xtickNames, rotation=45, fontsize=10)
# add to tuple
ax.legend( tuple(histList), tuple(legendList) , loc='upper left')
ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
plotFolderName = 'plots/crossDataSetMMDall/accuracy'+'/'+dataSet+'/'
checkAndCreateFolder(plotFolderName)
plotFileName = plotFolderName+dataSet+'_'+classifier+'.png'
plt.savefig(plotFileName, bbox_inches='tight')
plt.show()
plt.close()
accuracyFolderName = 'plots/crossDataSetMMDall/accuracyValues'+'/'+dataSet+'/'
checkAndCreateFolder(accuracyFolderName)
accuracyFileName = accuracyFolderName+dataSet+'_'+classifier+'.npy'
# save the image in some format
with open(accuracyFileName,'wb+') as fh:
np_save(fh, accuracyArray, allow_pickle=False)
sync(fh)
def _save(self, matrix: array):
print(f"Saving t-SNE data for {self.dims} dimensions...")
np_save(self.save_path_for(dims=self.dims, distance_name=self.distance_name), matrix)
if __name__ == '__main__':
patient = 5
dir_pri1 = './AT_tT1t_S_Patt'
dir_pri2 = '../FilesIN/Time_Courses/only_positives'
dir_pri3 = './fRMI_only_positives'
if not os.path.exists(dir_pri1):
print 'Pas de dossier ' + dir_pri1
quit()
if not os.path.exists(dir_pri2):
print 'Pas de dossier ' + dir_pri2
quit()
if not os.path.exists(dir_pri3):
os.mkdir(dir_pri3)
t0 = time()
pool = multiprocessing.Pool()
args = arange(0.50,15.01,0.5).tolist()
resultslist = pool.map(whatyouwant, args)
resultsarra = resultslist[0]
for ll in range(1, len(resultslist)):
resultsarra = append(resultsarra, resultslist[ll], axis=0)
np_save(dir_pri3 + '/correlations_S_p%i.npy' %patient, resultsarra)
print "Simulation took :",time()-t0,"s"
print "min= %.2f ; max= %.2f" %(resultsarra.min(), resultsarra.max())
imshow(abs(resultsarra), cmap=get_cmap('Blues'), interpolation='nearest')
show()
return W
if __name__ == '__main__':
dir_pri2 = './MatricesG30TVarXAda3'
if not os.path.exists(dir_pri2):
print 'Pas de dossier ' + dir_pri2
quit()
t0 = time()
pool = multiprocessing.Pool()
args = arange(0.50, 4.501, 0.2).tolist() # normW
results = pool.map(whatyouwant, args)
results = array(results)
np_save(dir_pri2 + '/results.npy', results)
print "Simulation took :", time() - t0, "s"
W = []
for j in range(results.shape[1]):
figure()
for i in range(results.shape[0]):
subplot(3, 7, i + 1)
title(i)
imshow(results[i, j], interpolation='nearest')
show()
mats = input()
for i in range(results.shape[0]):
if i in mats:
W.append(results[i, j])
epochs = 150
dims = 50
eta = 0.1
window_size = 1
minimum_count = 2
for model_name, Word2Vec_model in [
('Gensim', Word2Vec_gensim),
('Train_on_current', Word2Vec_train_on_current),
('Not_optimised', Word2Vec_no_optimisations),
('No_train_operator_input', Word2Vec_no_train_w_operator_input)
]:
for filename in os.listdir(corpus_dir):
if filename.endswith('.txt'):
fullpath = os.path.join(corpus_dir, filename)
logic_examples = SpacedLines(fullpath)
models_dict['{}-{}'.format(model_name,
filename)] = Word2Vec_model(
logic_examples,
sg=1,
min_count=minimum_count,
window=window_size,
size=dims,
alpha=eta,
iter=epochs)
print('Model {} file {} complete'.format(model_name, filename))
np_save('models_dict2.npy', models_dict)
print('Done!!')
labels_file = args.input_labels
output_dir = args.output_dir
dim_steps = [int(x) for x in args.dims.split(',')]
output_dir = args.output_dir
scripts_dir = args.scripts_dir
tag = args.tag
labels = np_load(labels_file)
cv_folds = BalancedKFold(labels, args.num_folds, n_iter=args.cv_iters)
i = 0
for k, (training, testing) in enumerate(cv_folds):
file_prefix = path.join(output_dir, 'CV_%d_' % k)
training_file = file_prefix+'training.npy'
np_save(training_file, training)
testing_file = file_prefix+'testing.npy'
np_save(testing_file, testing)
for num_dims in dim_steps:
if i >= args.max_scripts:
ind = i % args.max_scripts
script_file = path.join(scripts_dir, '%d.sh' % ind)
add_to_bash_script(script_file, data_matrix_file, labels_file, output_dir,
training_file, testing_file, num_dims, k, args.methods)
else:
script_file = path.join(scripts_dir, '%d.sh' % i)
make_bash_script(script_file, data_matrix_file, labels_file, output_dir,
training_file, testing_file, num_dims, k, args.methods)
i += 1
def save(self, arr, pth):
with open(pth, 'wb+') as fh:
np_save(fh, arr, allow_pickle=False)
sync(fh)
def main():
data = import_urls(redownload_data=False)
## Set parameter to True for initial download.
## Once data is present, set this to False to
## prevent re-downloading data.
X_10k, y_10k = None, None
X_60k, y_60k = None, None
for tag, num_items, raw_data in data:
#print(tag, num_items, raw_data.shape, np.min(raw_data), np.max(raw_data));
if num_items == 10000:
if tag == r'image': X_10k = raw_data
if tag == r'label': y_10k = raw_data
if num_items == 60000:
if tag == r'image': X_60k = raw_data
if tag == r'label': y_60k = raw_data
digits_nn = NN_digits(
X_10k, ## input data.
y_10k, ## output data.
3, ## 3 NN layers: Input, inter-layer, output.
[28 * 28, 25, 10])
## num of nodes for each layer.
if Grad_Descent_Method:
print("\nNeural Network XNOR - using GRADIENT DESCENT ITERATION\n",
"#" * 30, "\n")
# File location where learned weight is saved.
theta_file = CURRENT_PATH + r'/' + 'theta.npy'
if LOAD_PREV_THETAS:
flat_thetas = np_load(theta_file)
digits_nn.unflatten_Thetas(flat_thetas)
if CONTINUOUS_TRAINING:
digits_nn.train_NN()
np_save(theta_file, digits_nn.flatten_Thetas())
else:
digits_nn.train_NN()
np_save(theta_file, digits_nn.flatten_Thetas())
# Display final cost after learning iterations.
print("Final Cost J = ", digits_nn.J_cost(digits_nn.a[-1]))
if PLOT_COST:
# Plot the J Cost vs. # of iterations. J should coverge as iteration increases.
x_axis = range(digits_nn.J_cost_values)
y_axis = digits_nn.J_cost_values
plt.plot(x_axis, y_axis, label='J_cost vs. # of Iterations')
plt.show()
# Pick 10 samples randomly from 60k samples pool.
for i in range(0, 10):
ith_sample = randint(0, 60000 - 1)
x_input = X_60k[ith_sample].flatten()
y_val = y_60k[ith_sample]
test_res = digits_nn.H_funct(x_input)
top_three_answers = ""
for val, prob in test_res[1]:
top_three_answers += str(val) + ":" + str(prob) + "%, "
top_three_answers = top_three_answers[:-2]
print("y_val = ", y_val[0], " || Test result = ", test_res[0],
"TOP THREE: ", top_three_answers)
## If result is not as expected, display the image.
if not test_res[0] == y_val:
plot_image(X_60k[ith_sample])
### Calculate error rate of 60k test samples.
###
if COUNT_ERROR:
error_cnt = 0
for i in range(60000):
x_input = X_60k[i]
y_val = y_60k[i]
test_res = digits_nn.H_funct(x_input)
error_cnt += (test_res[0] != y_val)
error_rate = int(error_cnt / 6) / 100
print("ERROR RATE: ", error_rate, "%.")
if Minimize_funct_method:
print("\nNeural Network XNOR - using fmin_bfgs\n", "#" * 30, "\n")
digits_nn.init_thetas()
digits_nn.train_NN_with_fmin()
for i in range(0, 1000, 10):
x_input = X_60k[i].flatten()
y_val = y_60k[i]
test_res = digits_nn.H_funct(x_input)
print("y_val = ", y_val[0], " || Test result = ", test_res,
test_res == y_val)
print("\n")
return 0
def test(dataSet, generatedClass, numOfInstances):
'''
Inputs :
dataSets : List : Datasets for which samples are to be genrated
instances : List : Number of instances to be used from original dataset
classes : List : Classes for which samples are to be generated
Output :
File with 1000 compressed images generated by GAN
'''
modelFolder = 'models' + '/' + dataSet
modelFile = modelFolder+'/'+dataSet+'_'+str(generatedClass)+'_'+ \
str(numOfInstances)+'_'+str(getEpochs(numOfInstances))+'.pt'
print('Generating examples for Dataset: ' + dataSet + ' Class: ' +
str(generatedClass) + ' Instances: ' + str(numOfInstances) +
' Epochs: ' + str(getEpochs(numOfInstances)))
# load the model leannt during training
G = Generator(params.numInputChannels, params.numGenFilter,
params.numOutputChannels)
G.load_state_dict(torch.load(modelFile))
genImageConcat = np.empty(1)
for sample in range(params.numOfSamples):
noise = torch.FloatTensor(1, params.numInputChannels, 1, 1)
noise.normal_(0, 1)
if params.cuda:
G = G.cuda()
noise = noise.cuda()
noiseVariable = Variable(noise)
genImage = G(noiseVariable)
genImage = genImage.data
genImage = genImage.cpu()
genImage = genImage.numpy()
maxImage = np.max(genImage)
minImage = np.min(genImage)
#print maxImage, minImage
genImage = np.multiply(
np.divide((genImage - minImage), (maxImage - minImage)), 255.0)
maxImage = np.max(genImage)
minImage = np.min(genImage)
if sample == 0:
genImageConcat = genImage
elif sample >= 1:
genImageConcat = np.concatenate((genImageConcat, genImage), axis=0)
path = '../DCGAN/results'+'/'+'compressed'+'/'+dataSet+'/'+ dataSet + '_' \
+ str(generatedClass) + '_' + str(numOfInstances) + '.npy'
# save the image in some format
with open(path, 'wb+') as fh:
genImageConcat = np.squeeze(genImageConcat)
np_save(fh, genImageConcat, allow_pickle=False)
sync(fh)
if __name__=="__main__":
args = interface()
num_dims = args.num_dims
prefix = 'CV_%d' % (args.cv_fold_id)
techniques = get_methods(args.methods)
output_dir = args.output_dir
data_matrix = np_load(args.input_file)
labels = np_load(args.input_labels)
testing = np_load(args.testing_vector)
training = np_load(args.training_vector)
test_labels = labels[testing]
training_labels = labels[training]
test_matrix = data_matrix[testing, :]
training_matrix = data_matrix[training, :]
for technique, technique_name in techniques:
dim_redux = technique(n_components=num_dims)
file_label = '%s_%s_%s_' % (prefix,
technique_name,
str(num_dims))
file_prefix = path.join(output_dir, file_label)
txd_training_matrix = dim_redux.fit_transform(training_matrix, training_labels)
txd_test_matrix = dim_redux.transform(test_matrix)
np_save(file_prefix + 'txd_test_matrix', txd_test_matrix)
np_save(file_prefix + 'txd_training_matrix', txd_training_matrix)
def save(self, data):
validate_dtype(data, ndarray)
np_save(file=self.path, arr=data, allow_pickle=True)
def fit_model(model: torch.nn.Module, train_dl: DataLoader, val_dl: DataLoader,
opt: Optimizer, perplexity: int, n_epochs: int,
save_dir_path: str, epochs_to_save_after: int,
early_exaggeration: int, early_exaggeration_constant: int,
batch_size: int, dist_func_name: str, bin_search_tol: float,
bin_search_max_iter: int, min_allowed_sig_sq: float,
max_allowed_sig_sq: float, configuration_report: str) -> None:
"""
Fits t-SNE model
:param model: nn.Module instance
:param train_dl: data loader with points for training
:param val_dl: data loader with points for validation and early stopping
:param opt: optimizer instance
:param perplexity: perplexity
:param n_epochs: Number of epochs for training
:param save_dir_path: path to directory to save a trained model to
:param epochs_to_save_after: number of epochs to save a model after. If passed None,
model won't be saved at all
:param early_exaggeration: Number of first training cycles in which
exaggeration will be applied
:param early_exaggeration_constant: Constant by which p_joint is multiplied in early exaggeration
:param batch_size: Batch size for training
:param dist_func_name: Name of distance function for distance matrix.
Possible names: "euc", "jaccard", "cosine"
:param bin_search_tol: Tolerance threshold for binary search to obtain p_cond
:param bin_search_max_iter: Number of max iterations for binary search
:param min_allowed_sig_sq: Minimal allowed value for squared sigmas
:param max_allowed_sig_sq: Maximal allowed value for squared sigmas
:param configuration_report: Config of the model in string form for report purposes
:return:
"""
model_name = get_random_string(6)
train_batch_losses = []
train_epoch_losses = []
val_batch_losses = []
val_epoch_losses = []
for epoch in range(n_epochs):
epoch_start_time = datetime.datetime.now()
model.train()
for list_with_batch in tqdm(train_dl):
orig_points_batch, _ = list_with_batch
with torch.no_grad():
p_joint_in_batch = calc_p_joint_in_batch(
perplexity, orig_points_batch, dist_func_name,
bin_search_tol, bin_search_max_iter, min_allowed_sig_sq,
max_allowed_sig_sq)
opt.zero_grad()
embeddings = model(orig_points_batch)
q_joint_in_batch = get_q_joint(embeddings, "euc", alpha=1)
if early_exaggeration:
p_joint_in_batch *= early_exaggeration_constant
early_exaggeration -= 1
loss = loss_function(p_joint_in_batch, q_joint_in_batch)
train_batch_losses.append(loss.item())
loss.backward()
opt.step()
model.eval()
for val_list_with_batch in tqdm(val_dl):
val_orig_points_batch, _ = val_list_with_batch
with torch.no_grad():
p_joint_in_batch_val = calc_p_joint_in_batch(
perplexity, val_orig_points_batch, dist_func_name,
bin_search_tol, bin_search_max_iter, min_allowed_sig_sq,
max_allowed_sig_sq)
val_embeddings = model(val_orig_points_batch)
q_joint_in_batch_val = get_q_joint(val_embeddings, "euc", alpha=1)
loss_val = loss_function(p_joint_in_batch_val,
q_joint_in_batch_val)
val_batch_losses.append(loss_val.item())
train_epoch_loss = mean(train_batch_losses)
train_epoch_losses.append(train_epoch_loss)
val_epoch_loss = mean(val_batch_losses)
val_epoch_losses.append(val_epoch_loss)
train_batch_losses = []
val_batch_losses = []
epoch_end_time = datetime.datetime.now()
time_elapsed = epoch_end_time - epoch_start_time
# Report loss for epoch
print(
f'====> Epoch: {epoch + 1}. Time {time_elapsed}. Average loss: {train_epoch_loss:.4f}. Val loss: {val_epoch_loss:.4f}',
flush=True)
# Save model and loss history if needed
save_path = os.path.join(save_dir_path,
f"{model_name}_epoch_{epoch + 1}")
if epochs_to_save_after is not None and (
epoch + 1) % epochs_to_save_after == 0:
torch.save(model, save_path + ".pt")
with open(save_path + ".json", "w") as here:
json.dump(json.loads(configuration_report), here)
print('Model saved as %s' % save_path, flush=True)
if epochs_to_save_after is not None and epoch == n_epochs - 1:
loss_save_path = save_path + "_loss.npy"
np_save(
loss_save_path,
vstack((array(train_epoch_losses), array(val_epoch_losses))))
print("Loss history saved in", loss_save_path, flush=True)
