def main(args, param_grid=None, plot_confusion=False):
if param_grid is None:
param_grid = {}
# Read the train and test files.
train_filenames, train_labels = load_dataset(args.train_path)
test_filenames, test_labels = load_dataset(args.test_path)
# Compute the Dense SIFT descriptors for all the train and test images.
sift = DenseSIFT(memory=args.cache_path)
with Timer('Extract train descriptors'):
train_pictures = sift.compute(train_filenames)
with Timer('Extract test descriptors'):
test_pictures = sift.compute(test_filenames)
train_data = np.array([p.to_array() for p in train_pictures], copy=False)
test_data = np.array([p.to_array() for p in test_pictures], copy=False)
le = LabelEncoder()
le.fit(train_labels)
train_labels = le.transform(train_labels)
test_labels = le.transform(test_labels)
# Create processing pipeline and run cross-validation.
transformer = SpatialPyramid(n_clusters=760,
n_samples=100000,
n_levels=2,
norm='power')
scaler = StandardScaler(copy=False)
classifier = SVC(C=1, kernel=histogram_intersection_kernel, gamma=.001)
memory = Memory(location=args.cache_path, verbose=1)
pipeline = Pipeline(memory=memory,
steps=[('transformer', transformer),
('scaler', scaler), ('classifier', classifier)])
cv = GridSearchCV(pipeline,
param_grid,
n_jobs=-1,
cv=3,
refit=True,
verbose=11,
return_train_score=True)
with Timer('Train'):
cv.fit(train_data, train_labels)
with Timer('Test'):
accuracy = cv.score(test_data, test_labels)
print('Best params: {}'.format(cv.best_params_))
print('Accuracy: {}'.format(accuracy))
if plot_confusion:
plot_confusion_matrix(le.inverse_transform(test_labels),
le.inverse_transform(cv.predict(test_data)))
return pandas.DataFrame.from_dict(cv.cv_results_)
def run(args):
# Read the train and test files.
train_filenames, train_labels = load_dataset(args.train_path)
test_filenames, test_labels = load_dataset(args.test_path)
results = []
for m, nf, ss in product(args.method, process_arg(args.n_features),
process_arg(args.step_size)):
print('method: {}, n_features: {}, step_size: {}'.format(m, nf, ss))
if m == 'sift':
method = SIFT(n_features=nf)
elif m == 'dense_sift':
method = DenseSIFT(step_size=ss)
else:
raise Exception('Invalid method')
# Compute the SIFT descriptors for all the train images.
with Timer('extract train descriptors'):
train_descriptors = method.compute(train_filenames)
# Compute the test descriptors.
with Timer('extract test descriptors'):
test_descriptors = method.compute(test_filenames)
for k, n, d in product(process_arg(args.n_clusters),
process_arg(args.n_neighbors), args.distance):
print('n_clusters: {}, n_neighbors: {}, distance: {}'.format(
k, n, d))
# Train the classifier and compute accuracy of the model.
classifier = Classifier(k, n, d)
with Timer('train'):
classifier.train(train_descriptors, train_labels)
with Timer('test'):
accuracy = classifier.test(test_descriptors, test_labels)
results.append((m, d, nf, ss, k, n, accuracy))
# Optionally plot confusion matrix.
if args.confusion_matrix:
plot_confusion_matrix(test_labels,
classifier.predict(test_descriptors))
return pandas.DataFrame(results,
columns=[
"method", "distance", "n_features",
"step_size", "n_clusters", "n_neighbors",
"accuracy"
])
def learning(args):
initial_thetas = load_thetas()
kms, prices = load_dataset()
normalized_plots = _normalize_dataset(kms, prices)
final_thetas = gradient_descent(
*initial_thetas,
normalized_plots,
args.learning,
args.iterations,
)
csv.write_thetas(final_thetas)
if args.precision:
print(
f"The MSE between predicted and actual is {cost_fn(*final_thetas, normalized_plots)}"
)
if args.show:
plot.dataset(kms, prices)
price_estimations = [
denormalize(estimate_price(*final_thetas, data.mileage), prices)
for data in normalized_plots
]
plot.linear_regression(kms, price_estimations)
plot.show()
def run(args, kwargs):
args.snap_dir = snap_dir = \
'snapshots/discrete_logisticcifar10_flows_2_levels_3__2019-09-27_13_08_49/'
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
final_model = torch.load(snap_dir + 'a.model')
if hasattr(final_model, 'module'):
final_model = final_model.module
from models.backround import SmoothRound
for module in final_model.modules():
if isinstance(module, SmoothRound):
module._round_decay = 1.
exp_dir = snap_dir + 'partials/'
os.makedirs(exp_dir, exist_ok=True)
images = []
with torch.no_grad():
for data, _ in test_loader:
if args.cuda:
data = data.cuda()
for i in range(len(data)):
_, _, _, pz, z, pys, ys, ldj = final_model.forward(data[i:i +
1])
for j in range(len(ys) + 1):
x_recon = final_model.inverse(z, ys[len(ys) - j:])
images.append(x_recon.float())
if i == 10:
break
break
for j in range(len(ys) + 1):
grid = make_grid(torch.stack(images[j::len(ys) + 1], dim=0).squeeze(),
nrow=11,
padding=0,
normalize=True,
range=None,
scale_each=False,
pad_value=0)
imageio.imwrite(exp_dir + 'loaded{j}.png'.format(j=j),
grid.cpu().numpy().transpose(1, 2, 0))
def prediction(args):
thetas = load_thetas()
kms, prices = load_dataset()
norm_mileage = normalize(args.mileage, kms)
estimation = estimate_price(*thetas, norm_mileage)
if sum(thetas) != 0:
estimation = denormalize(estimation, prices)
print(f"Estimation for {args.mileage} is {round(estimation, 2)}")
def build_with(filename, oldtype, nres):
newp = convert_netw_type(oldtype)
type, wbits, abits = get_netw_type_wbits_abits(newp)
print(type, wbits, abits)
cf = {
"architecture": "RESNET",
"dim": 32,
"channels": 3,
"classes": 10,
"nres": nres,
"kernel_initializer": 'he_normal',
"kernel_regularizer": 1e-4,
"dataset": "CIFAR-10",
"network_type": type,
"wbits": wbits,
"abits": abits,
"pfilt": 1
}
cf = obj(cf)
model = build_model(cf)
wname = filename
model.load_weights(wname)
# loss = 'categorical_crossentropy'
# model.compile(loss=loss, optimizer='Adam', metrics=['accuracy'])
train_data, val_data, test_data = load_dataset("CIFAR-10", cf)
for la in range(20, 21):
layer_name = f"conv2d_{la+1}"
intermediate_layer_model = Model(
inputs=model.input, outputs=model.get_layer(layer_name).output)
# import matplotlib.pyplot as plt
# import matplotlib.image as mpimg
# imgplot = plt.imshow(test_data.X[0])
# plt.show()
# score = intermediate_layer_model.predict(np.array([np.ones(test_data.X[0].shape)]), verbose=0)
score = intermediate_layer_model.predict(test_data.X, verbose=1)
# print(score[0])
for elem in range(score.shape[0]):
for i in range(score.shape[1]):
for j in range(score.shape[2]):
for k in range(score.shape[3]):
if abs(score[elem][i][j][k]) > 63:
print(
f"SO SAAAAAAAAAd: element:{elem} layer:{la+1} value:{abs(score[elem][i][j][k])}"
)
break
# print(f"{score[0][i][j][k]:2.1}", end=' ')
# print('')
# print('Test loss:', score[0])
# print('Test accuracy:', score[1])
# print(score)
return score #[1]
def classify_personalized(file_path):
X_train, y_train, X_test, y_test = load_dataset()
num_patient = len(X_train)
random_seed = 0;
n_estimators = 50;
models = {'LDA': LinearDiscriminantAnalysis(solver='svd'),
# 'Decision Tree': DecisionTreeClassifier(max_depth=None,
# random_state=random_seed),
'Bagging': BaggingClassifier(DecisionTreeClassifier(max_depth=15), n_estimators=n_estimators,
random_state=random_seed),
'Random Forest': RandomForestClassifier(n_estimators=n_estimators,
random_state=random_seed),
'Extremely Randomized Trees': ExtraTreesClassifier(n_estimators=n_estimators,
random_state=random_seed),
'Ada Boost': AdaBoostClassifier(DecisionTreeClassifier(max_depth=15), n_estimators=n_estimators,
random_state=random_seed),
'SVM_tuned': SVC(kernel='linear', C=1, class_weight="balanced", gamma='auto', probability=True),
'KNN': KNeighborsClassifier(n_neighbors=40)
# 'Logistic Regression': LogisticRegression(solver='liblinear', multi_class='ovr', C=1.0,
# random_state=random_seed),
}
target_names = ['class 0', 'class 1', 'class 2', 'class 3', 'class 4', 'class 5', 'class 6', 'class 7']
try:
with open(file_path, 'rb') as f:
scores = pickle.load(f)
except FileNotFoundError:
scores = {}
for patient in range(num_patient):
if patient in scores:
print(patient)
continue
print("patient:", patient)
# x_train = X_train[patient].copy()
# y_train = y_train[patient].copy()
# x_test = X_test[patient].copy()
# y_test = y_test[patient].copy()
score = {}
# for model_name in models:
#params_list = [list(models.values()), x_train.copy(), y_train.copy(), x_test.copy(), y_test.copy(), list(models.keys())]
params_list = [list(models.values()), X_train[patient], y_train[patient], X_test[patient], y_test[patient],
list(models.keys())]
results = customMultiprocessing(model_fit, params_list, pool_size=8)
score = {}
for result in results:
score.update(result)
scores[patient] = score
print("Save patient {}".format(patient))
with open(file_path, 'wb') as f:
pickle.dump(scores, f)
return
def shap_values_tree(path_shap):
X_train, y_train, X_test, y_test = load_dataset()
num_patient = len(X_train)
random_seed = 0
n_estimators = 50
models = {
'ExtremelyRandomizedTrees':
ExtraTreesClassifier(n_estimators=n_estimators,
random_state=random_seed),
}
for model_name in models.keys():
print(model_name)
#for each model train the model for each patient and compute shap values
for patient in range(3, num_patient):
shap_list = []
print((patient))
#training
model = models[model_name]
model.fit(X_train[patient], y_train[patient])
#create explainer
shap_explainer = shap.TreeExplainer(
model, X_train[patient].iloc[0:1000, :])
print(X_test[patient].shape)
shap_values = shap_explainer.shap_values(X_test[patient])
for i in range(len(shap_values)):
shap_df = pd.DataFrame(data=shap_values[i],
columns=X_test[i].columns.values)
shap_list.append(shap_df)
path_img_bar = path_shap + "model{}_patient{}_allclasses_background_data.png".format(
model_name, patient)
plt.figure()
shap.summary_plot(shap_values,
X_test[patient],
plot_type="bar",
show=False)
plt.savefig(path_img_bar)
with open(
'../resources/shap_{}_patient_{}_background_data.pkl'.
format(model_name, patient), 'wb') as f:
pickle.dump(shap_list, f)
# for value in range(len(shap_values)):
# path_img_dot = path_shap + "dot_patient{}_class{}.png".format(patient, value)
# plt.figure()
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="dot", show=False)
# plt.savefig(path_img_dot)
# plt.close()
# plt.figure()
# path_img_bar = path_shap + "bar_patient{}_class{}.png".format(patient, value)
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="bar", show=False)
# plt.savefig(path_img_bar)
# plt.close()
return
def run(args, kwargs):
args.model_signature = str(datetime.datetime.now())[0:19]
model_name = args.dataset_name + '_' + args.model_name + '_' + args.prior + '(K_' + str(args.number_components) + ')' + '_wu(' + str(args.warmup) + ')' + '_z1_' + str(args.z1_size) + '_z2_' + str(args.z2_size)
# DIRECTORY FOR SAVING
snapshots_path = 'snapshots/'
dir = snapshots_path + args.model_signature + '_' + model_name + '/'
if not os.path.exists(dir):
os.makedirs(dir)
# LOAD DATA=========================================================================================================
print('load data')
# loading data
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# CREATE MODEL======================================================================================================
print('create model')
# importing model
if args.model_name == 'vae':
from models.VAE import VAE
elif args.model_name == 'hvae_2level':
from models.HVAE_2level import VAE
elif args.model_name == 'convhvae_2level':
from models.convHVAE_2level import VAE
elif args.model_name == 'pixelhvae_2level':
from models.PixelHVAE_2level import VAE
else:
raise Exception('Wrong name of the model!')
model = VAE(args)
if args.cuda:
model.cuda()
optimizer = AdamNormGrad(model.parameters(), lr=args.lr)
# optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ======================================================================================================================
print(args)
with open('vae_experiment_log.txt', 'a') as f:
print(args, file=f)
# ======================================================================================================================
print('perform experiment')
from utils.perform_experiment import experiment_vae
experiment_vae(args, train_loader, val_loader, test_loader, model, optimizer, dir, model_name = args.model_name)
# ======================================================================================================================
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-')
with open('vae_experiment_log.txt', 'a') as f:
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n', file=f)
def run(args, kwargs):
args.model_signature = str(datetime.datetime.now())[0:19]
model_name = args.dataset_name + '_' + args.model_name + '_' + args.prior + '(K_' + str(args.number_components) + ')' + '_wu(' + str(args.warmup) + ')' + '_z1_' + str(args.z1_size) + '_z2_' + str(args.z2_size)
# DIRECTORY FOR SAVING
snapshots_path = 'snapshots/'
dir = snapshots_path + args.model_signature + '_' + model_name + '/'
if not os.path.exists(dir):
os.makedirs(dir)
# LOAD DATA=========================================================================================================
print('load data')
# loading data
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# CREATE MODEL======================================================================================================
print('create model')
# importing model
if args.model_name == 'vae':
from models.VAE import VAE
elif args.model_name == 'hvae_2level':
from models.HVAE_2level import VAE
elif args.model_name == 'convhvae_2level':
from models.convHVAE_2level import VAE
elif args.model_name == 'pixelhvae_2level':
from models.PixelHVAE_2level import VAE
else:
raise Exception('Wrong name of the model!')
model = VAE(args)
if args.cuda:
model.cuda()
optimizer = AdamNormGrad(model.parameters(), lr=args.lr)
# ======================================================================================================================
print(args)
with open('vae_experiment_log.txt', 'a') as f:
print(args, file=f)
# ======================================================================================================================
print('perform experiment')
from utils.perform_experiment import experiment_vae
experiment_vae(args, train_loader, val_loader, test_loader, model, optimizer, dir, model_name = args.model_name)
# ======================================================================================================================
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-')
with open('vae_experiment_log.txt', 'a') as f:
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n', file=f)
def main():
args = parser.parse_args()
args.cuda = torch.cuda.is_available()
args.break_epoch = False
snap_dir = args.snap_dir = join('snapshots', args.snap_dir) + '/'
train_loader, val_loader, test_loader, args = load_dataset(args)
final_model = torch.load(snap_dir + 'a.model', map_location='cpu')
if args.cuda:
final_model = final_model.cuda()
# Just for timing at the moment.
with torch.no_grad():
final_model.eval()
timing_results = []
for i in range(100):
torch.cuda.synchronize()
start = time.time()
x_sample = final_model.sample(n_samples=100)
torch.cuda.synchronize()
duration = time.time() - start
timing_results.append(duration)
print('Timings: ', timing_results)
print('Mean time:', np.mean(timing_results))
plot_samples(final_model, args, epoch=9999, bpd=0.0)
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
final_model = torch.nn.DataParallel(final_model, dim=0)
test_bpd = evaluate(test_loader, final_model, args)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'FINAL \ttest negative elbo bpd {:.4f}'.format(test_bpd)
print(msg)
print(msg, file=ff)
test_bpd = evaluate(test_loader, final_model, args, iw_samples=1000)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'FINAL \ttest negative log_px bpd {:.4f}'.format(test_bpd)
print(msg)
print(msg, file=ff)
def shap_values_svm(path_shap):
X_train, y_train, X_test, y_test = load_dataset()
num_patient = len(X_train)
model = SVC(kernel='linear',
C=1,
class_weight="balanced",
gamma='auto',
probability=True)
for patient in range(num_patient):
model.fit(X_train[patient], y_train[patient])
explainer = shap.SamplingExplainer(model.predict_proba,
X_train[patient].iloc[0:100, :])
shap_values = explainer.shap_values(X_test[patient])
with open(
'../resources/shap_SVM/shap_svm_patient_{}_SVM.pkl'.format(
patient), 'wb') as f:
pickle.dump(shap_values, f)
path_img_bar = path_shap + "patient{}_svm.png".format(patient)
plt.figure()
shap.summary_plot(shap_values,
X_test[patient],
plot_type="bar",
show=False)
plt.savefig(path_img_bar)
# for value in range(len(shap_values)):
# path_img_dot = path_shap + "dot_patient{}_class{}.png".format(patient, value)
# plt.figure()
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="dot", show=False)
# plt.savefig(path_img_dot)
# plt.close()
# plt.figure()
# path_img_bar = path_shap + "bar_patient{}_class{}.png".format(patient, value)
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="bar", show=False)
# plt.savefig(path_img_bar)
# plt.close()
return
def shap_values(path_shap):
X_train, y_train, X_test, y_test = load_dataset()
num_patient = len(X_train)
random_seed = 0
n_estimators = 100
model = ExtraTreesClassifier(n_estimators=n_estimators,
random_state=random_seed)
shap_list = []
for patient in range(num_patient):
model.fit(X_train[patient], y_train[patient])
shap_explainer = shap.TreeExplainer(model)
shap_values = shap_explainer.shap_values(X_test[patient])
# path_img_bar = path_shap + "patient{}_allclasses.png".format(patient)
# plt.figure()
# shap.summary_plot(shap_values, X_test[patient], plot_type="bar", show=False)
# plt.savefig(path_img_bar)
for i in range(len(shap_values)):
shap_df = pd.DataFrame(data=shap_values[i],
columns=X_test[i].columns.values)
shap_list.append(shap_df)
with open(
'../resources/shap_extratrees_patient_{}_tmp.pkl'.format(
patient), 'wb') as f:
pickle.dump(shap_list, f)
# for value in range(len(shap_values)):
# path_img_dot = path_shap + "dot_patient{}_class{}.png".format(patient, value)
# plt.figure()
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="dot", show=False)
# plt.savefig(path_img_dot)
# plt.close()
# plt.figure()
# path_img_bar = path_shap + "bar_patient{}_class{}.png".format(patient, value)
# shap.summary_plot(shap_values[value], X_test[patient], plot_type="bar", show=False)
# plt.savefig(path_img_bar)
# plt.close()
return
def run(args, kwargs):
# Only for the residual networks (resflow/sylvester) comparison.
args.grad_norm_enabled = True
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
model, model_sample, optimizer, scheduler = setup(args)
writer = tensorboardX.SummaryWriter(logdir=args.snap_dir)
snap_dir = args.snap_dir
# ==================================================================================================================
# TRAINING
# ==================================================================================================================
train_bpd = []
val_bpd = []
# for early stopping
best_val_bpd = np.inf
best_train_bpd = np.inf
epoch = 0
train_times = []
model.eval()
model.train()
starting_epoch = 1
if args.restart_from_epoch is not None:
starting_epoch = args.restart_from_epoch
for epoch in range(starting_epoch, args.epochs + 1):
t_start = time.time()
tr_bpd = train(epoch, train_loader, model, optimizer, args)
scheduler.step()
train_bpd.append(tr_bpd)
writer.add_scalar('train bpd', train_bpd[-1], epoch)
train_times.append(time.time() - t_start)
print('One training epoch took %.2f seconds' % (time.time() - t_start))
if epoch in [1, 5, 10] or epoch % args.evaluate_interval_epochs == 0:
tr_bpd = evaluate(train_loader, model, args, iw_samples=1)
v_bpd = evaluate(val_loader, model, args, iw_samples=1)
# Logging message.
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttrain bpd {:.3f}\tval bpd {:.3f}\t'.format(
epoch, tr_bpd, v_bpd)
print(msg, file=ff)
plot_samples(model_sample, args, epoch, v_bpd)
val_bpd.append(v_bpd)
writer.add_scalar('val bpd', v_bpd, epoch)
# Model save based on val performance
if v_bpd < best_val_bpd:
best_train_bpd = tr_bpd
best_val_bpd = v_bpd
try:
if hasattr(model, 'module'):
torch.save(model.module, snap_dir + 'a.model')
else:
torch.save(model, snap_dir + 'a.model')
torch.save(optimizer, snap_dir + 'a.optimizer')
print('->model saved<-')
except:
print('Saving was unsuccessful.')
print('(BEST: train bpd {:.4f}, val bpd {:.4f})\n'.format(
best_train_bpd, best_val_bpd))
if math.isnan(v_bpd):
raise ValueError('NaN encountered!')
# training time per epoch
train_times = np.array(train_times)
mean_train_time = np.mean(train_times)
std_train_time = np.std(train_times, ddof=1)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_train_time, std_train_time))
# ========================================================================
# EVALUATION
# ========================================================================
final_model = torch.load(snap_dir + 'a.model')
test_bpd = evaluate(test_loader, final_model, args)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttest negative elbo bpd {:.4f}'.format(epoch, test_bpd)
print(msg, file=ff)
if 'residual' in args.model_type:
print('Importance weighted eval needs exact determinants.')
else:
test_bpd = evaluate(test_loader, final_model, args, iw_samples=1000)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttest negative log_px bpd {:.4f}'.format(
epoch, test_bpd)
print(msg, file=ff)
def run(args, kwargs):
print('\nMODEL SETTINGS: \n', args, '\n')
print("Random Seed: ", args.manual_seed)
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
args.model_signature = str(datetime.datetime.now())[0:19].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
snapshots_path = os.path.join(args.out_dir, 'vae_' + args.dataset + '_')
snap_dir = snapshots_path + args.flow + '_gpunum_' + str(args.gpu_num)
if args.flow != 'no_flow':
snap_dir += '_' + 'num_flows_' + str(args.num_flows)
if args.flow == 'orthogonal':
snap_dir = snap_dir + '_num_vectors_' + str(args.num_ortho_vecs)
elif args.flow == 'householder':
snap_dir = snap_dir + '_num_householder_' + str(args.num_householder)
elif args.flow == 'iaf':
snap_dir = snap_dir + '_madehsize_' + str(args.made_h_size)
snap_dir = snap_dir + '__' + args.model_signature + '/'
args.snap_dir = snap_dir
if not os.path.exists(snap_dir):
os.makedirs(snap_dir)
# SAVING
torch.save(args, snap_dir + args.flow + '.config')
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# ==================================================================================================================
# SELECT MODEL
# ==================================================================================================================
# flow parameters and architecture choice are passed on to model through args
if args.flow == 'no_flow':
model = VAE.VAE(args)
elif args.flow == 'planar':
model = VAE.PlanarVAE(args)
elif args.flow == 'iaf':
model = VAE.IAFVAE(args)
elif args.flow == 'orthogonal':
model = VAE.OrthogonalSylvesterVAE(args)
elif args.flow == 'householder':
model = VAE.HouseholderSylvesterVAE(args)
elif args.flow == 'triangular':
model = VAE.TriangularSylvesterVAE(args)
else:
raise ValueError('Invalid flow choice')
if args.cuda:
print("Model on GPU")
model.cuda()
print(model)
optimizer = optim.Adamax(model.parameters(), lr=args.learning_rate, eps=1.e-7)
# ==================================================================================================================
# TRAINING
# ==================================================================================================================
train_loss = []
val_loss = []
# for early stopping
best_loss = np.inf
best_bpd = np.inf
e = 0
epoch = 0
train_times = []
for epoch in range(1, args.epochs + 1):
t_start = time.time()
tr_loss = train(epoch, train_loader, model, optimizer, args)
train_loss.append(tr_loss)
train_times.append(time.time()-t_start)
print('One training epoch took %.2f seconds' % (time.time()-t_start))
v_loss, v_bpd = evaluate(val_loader, model, args, epoch=epoch)
val_loss.append(v_loss)
# early-stopping
if v_loss < best_loss:
e = 0
best_loss = v_loss
if args.input_type != 'binary':
best_bpd = v_bpd
print('->model saved<-')
torch.save(model, snap_dir + args.flow + '.model')
# torch.save(model, snap_dir + args.flow + '_' + args.architecture + '.model')
elif (args.early_stopping_epochs > 0) and (epoch >= args.warmup):
e += 1
if e > args.early_stopping_epochs:
break
if args.input_type == 'binary':
print('--> Early stopping: {}/{} (BEST: loss {:.4f})\n'.format(e, args.early_stopping_epochs, best_loss))
else:
print('--> Early stopping: {}/{} (BEST: loss {:.4f}, bpd {:.4f})\n'.format(e, args.early_stopping_epochs,
best_loss, best_bpd))
if math.isnan(v_loss):
raise ValueError('NaN encountered!')
train_loss = np.hstack(train_loss)
val_loss = np.array(val_loss)
plot_training_curve(train_loss, val_loss, fname=snap_dir + '/training_curve_%s.pdf' % args.flow)
# training time per epoch
train_times = np.array(train_times)
mean_train_time = np.mean(train_times)
std_train_time = np.std(train_times, ddof=1)
print('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time))
# ==================================================================================================================
# EVALUATION
# ==================================================================================================================
test_score_file = snap_dir + 'test_scores.txt'
with open('experiment_log.txt', 'a') as ff:
print(args, file=ff)
print('Stopped after %d epochs' % epoch, file=ff)
print('Average train time per epoch: %.2f +/- %.2f' % (mean_train_time, std_train_time), file=ff)
final_model = torch.load(snap_dir + args.flow + '.model')
if args.testing:
validation_loss, validation_bpd = evaluate(val_loader, final_model, args)
test_loss, test_bpd = evaluate(test_loader, final_model, args, testing=True)
with open('experiment_log.txt', 'a') as ff:
print('FINAL EVALUATION ON VALIDATION SET\n'
'ELBO (VAL): {:.4f}\n'.format(validation_loss), file=ff)
print('FINAL EVALUATION ON TEST SET\n'
'NLL (TEST): {:.4f}\n'.format(test_loss), file=ff)
if args.input_type != 'binary':
print('FINAL EVALUATION ON VALIDATION SET\n'
'ELBO (VAL) BPD : {:.4f}\n'.format(validation_bpd), file=ff)
print('FINAL EVALUATION ON TEST SET\n'
'NLL (TEST) BPD: {:.4f}\n'.format(test_bpd), file=ff)
else:
validation_loss, validation_bpd = evaluate(val_loader, final_model, args)
# save the test score in case you want to look it up later.
_, _ = evaluate(test_loader, final_model, args, testing=True, file=test_score_file)
with open('experiment_log.txt', 'a') as ff:
print('FINAL EVALUATION ON VALIDATION SET\n'
'ELBO (VALIDATION): {:.4f}\n'.format(validation_loss), file=ff)
if args.input_type != 'binary':
print('FINAL EVALUATION ON VALIDATION SET\n'
'ELBO (VAL) BPD : {:.4f}\n'.format(validation_bpd), file=ff)
cfg = arguments.config_path
cf = Config(cfg, cmd_args=arguments.override)
# if necessary, only use the CPU for debugging
if cf.cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = ""
else:
os.environ["CUDA_VISIBLE_DEVICES"] = cf.cuda
# ## Construct the network
print('Construct the Network\n')
model = build_model(cf)
print('loading data\n')
train_data, val_data, test_data = load_dataset(cf.dataset, cf)
print('setting up the network and creating callbacks\n')
checkpoint = ModelCheckpoint(cf.out_wght_path,
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='max',
period=1)
tensorboard = TensorBoard(log_dir=str(cf.tensorboard_name),
histogram_freq=0,
write_graph=True,
write_images=False)
callbacks = [checkpoint, tensorboard, MyEarlyStopping()]
# if True:
# def lr_schedule(epoch, lr):
def run(args, kwargs):
# LOAD DATA=========================================================================================================
print('load data')
checkpoints_dir, results_dir = create_dirNames(args)
# loading data
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# CREATE MODEL======================================================================================================
print('create model')
# importing model
if args.model_name == 'vae':
from models.VAE import VAE
model = VAE(args)
elif args.model_name == 'conv_vae':
from models.conv_vae import VAE
model = VAE(args)
elif args.model_name == 'hvae_2level':
from models.HVAE_2level import VAE
model = VAE(args)
elif args.model_name == 'convhvae_2level':
from models.convHVAE_2level import VAE
model = VAE(args)
elif args.model_name == 'convhvae_2level-smim':
from models.convHVAE_2level import SymMIM as VAE
model = VAE(args)
elif args.model_name == 'MLP_wae':
from models.MLP_wae import WAE
model = WAE(args)
elif args.model_name == 'conv_wae':
from models.conv_wae import WAE
model = WAE(args)
else:
raise Exception('Wrong name of the model!')
#model = VAE(args)
if args.cuda:
model.cuda()
optimizer = AdamNormGrad(model.parameters(),
lr=args.lr,
betas=(args.beta1, 0.999))
# ======================================================================================================================
print(args)
# ======================================================================================================================
print('perform experiment')
if args.trainflag:
if args.model_name == 'MLP_wae' or args.model_name == 'conv_wae' or args.model_name == 'Pixel_wae' or args.model_name == 'conv_wae_2level':
train_wae(args, train_loader, val_loader, model, optimizer,
checkpoints_dir, results_dir)
else:
train_vae(args, train_loader, val_loader, test_loader, model,
optimizer, checkpoints_dir, results_dir)
if args.testflag:
test(args, train_loader, test_loader, model, checkpoints_dir,
results_dir)
def run(args, kwargs):
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# args.snap_dir = snap_dir = \
# '/u/scottcao/scratch/idf/discrete_logisticoi_flows_8_levels_4__2020-02-23_12_29_53/'
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
_, _, test_loader, args = load_dataset(args, **kwargs)
final_model = torch.load(args.snap_dir + 'a.model')
print(f"Number of params: {count_params(final_model)}")
if hasattr(final_model, 'module'):
final_model = final_model.module
final_model = final_model.cuda()
sizes = []
errors = []
bpds = []
acc = timer.TimeAccumulator()
t = 0
with torch.no_grad():
for i, data in enumerate(test_loader):
data = data.squeeze(0)
with acc.execute():
if args.cuda:
data = data.cuda()
state, state_sizes, bpd, error = \
encode_images(data, final_model, decode=not args.no_decode)
errors += [error]
bpds.extend(bpd)
sizes.extend(state_sizes)
t += len(data)
print('Examples: {}/{} bpd compression: {:.3f} error: {},'
' analytical bpd {:.3f} time: {:.3f}'.format(
i,
len(test_loader.dataset),
np.sum(sizes) / np.prod(data.size()[1:]) / t,
np.sum(errors),
np.mean(bpds),
acc.mean_time_spent(),
),
end="\r")
# if args.no_decode:
# print('Not testing decoding.')
# else:
# print('Error: {}'.format(np.sum(errors)))
print()
print('Final bpd: {:.3f} error: {}'.format(
np.mean(sizes) / np.prod(data.size()[1:]), np.sum(errors)))
print('Took {:.3f} seconds / example'.format(acc.mean_time_spent()))
def loadDataset(cf):
if cf.dataset == "MNIST":
return load_dataset(cf.dataset, 50000, 55000)
else:
return load_dataset(cf.dataset, 45000, 50000)
def run(args, kwargs):
print('\nMODEL SETTINGS: \n', args, '\n')
print("Random Seed: ", args.manual_seed)
if 'imagenet' in args.dataset and args.evaluate_interval_epochs > 5:
args.evaluate_interval_epochs = 5
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
args.model_signature = str(datetime.datetime.now())[0:19].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
snapshots_path = os.path.join(
args.out_dir, args.variable_type + '_' + args.distribution_type + args.dataset)
snap_dir = snapshots_path
snap_dir += '_' + 'flows_' + \
str(args.n_flows) + '_levels_' + str(args.n_levels)
snap_dir = snap_dir + '__' + args.model_signature + '/'
args.snap_dir = snap_dir
if not os.path.exists(snap_dir):
os.makedirs(snap_dir)
with open(snap_dir + 'log.txt', 'a') as ff:
print('\nMODEL SETTINGS: \n', args, '\n', file=ff)
# SAVING
torch.save(args, snap_dir + '.config')
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# ==================================================================================================================
# SELECT MODEL
# ==================================================================================================================
# flow parameters and architecture choice are passed on to model through args
print(args.input_size)
import models.Model as Model
model = Model.Model(args)
args.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.set_temperature(args.temperature)
model.enable_hard_round(args.hard_round)
model_sample = model
# ====================================
# INIT
# ====================================
# data dependend initialization on CPU
for batch_idx, data in enumerate(train_loader):
model(data)
break
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = torch.nn.DataParallel(model, dim=0)
model.to(args.device)
def lr_lambda(epoch):
return min(1., (epoch+1) / args.warmup) * np.power(args.lr_decay, epoch)
optimizer = optim.Adamax(
model.parameters(), lr=args.learning_rate, eps=1.e-7)
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lr_lambda, last_epoch=-1)
# ==================================================================================================================
# TRAINING
# ==================================================================================================================
train_bpd = []
val_bpd = []
# for early stopping
best_val_bpd = np.inf
best_train_bpd = np.inf
epoch = 0
train_times = []
model.eval()
model.train()
for epoch in range(1, args.epochs + 1):
t_start = time.time()
scheduler.step()
tr_loss, tr_bpd = train(epoch, train_loader, model, optimizer, args)
train_bpd.append(tr_bpd)
train_times.append(time.time()-t_start)
print('One training epoch took %.2f seconds' % (time.time()-t_start))
if epoch < 25 or epoch % args.evaluate_interval_epochs == 0:
v_loss, v_bpd = evaluate(
train_loader, val_loader, model, model_sample, args,
epoch=epoch, file=snap_dir + 'log.txt')
val_bpd.append(v_bpd)
# Model save based on TRAIN performance (is heavily correlated with validation performance.)
if np.mean(tr_bpd) < best_train_bpd:
best_train_bpd = np.mean(tr_bpd)
best_val_bpd = v_bpd
torch.save(model, snap_dir + 'a.model')
torch.save(optimizer, snap_dir + 'a.optimizer')
print('->model saved<-')
print('(BEST: train bpd {:.4f}, test bpd {:.4f})\n'.format(
best_train_bpd, best_val_bpd))
if math.isnan(v_loss):
raise ValueError('NaN encountered!')
train_bpd = np.hstack(train_bpd)
val_bpd = np.array(val_bpd)
# training time per epoch
train_times = np.array(train_times)
mean_train_time = np.mean(train_times)
std_train_time = np.std(train_times, ddof=1)
print('Average train time per epoch: %.2f +/- %.2f' %
(mean_train_time, std_train_time))
# ==================================================================================================================
# EVALUATION
# ==================================================================================================================
final_model = torch.load(snap_dir + 'a.model')
test_loss, test_bpd = evaluate(
train_loader, test_loader, final_model, final_model, args,
epoch=epoch, file=snap_dir + 'test_log.txt')
print('Test loss / bpd: %.2f / %.2f' % (test_loss, test_bpd))
model.train()
if __name__ == '__main__':
ckpt = torch.load('expm/' + args.load_dict)
opt = args
args = ckpt['args']
args.save = 'expmplot/' + opt.load_dict
utils.makedirs(args.save)
utils.makedirs(args.save + '/samplescolumn')
logger = utils.get_logger(logpath=os.path.join(args.save, 'logs'),
filepath=os.path.abspath(__file__))
train_loader, val_loader, test_loader, args = load_dataset(
ckpt['args'], **kwargs)
#if args.flow == 'cnf_rank':
# model = CNFVAE.AmortizedLowRankCNFVAE(args).to(device)
#elif args.flow == 'cnf_bias':
# model = CNFVAE.AmortizedBiasCNFVAE(args).to(device)
#elif args.flow == 'cnf_hyper':
# model = CNFVAE.HypernetCNFVAE(args).to(device)
#elif args.flow == 'cnf_lyper':
# model = CNFVAE.LypernetCNFVAE(args).to(device)
#elif args.flow == 'sylvester':
model = VAE.HouseholderSylvesterVAE(ckpt['args']).to(device)
model.load_state_dict(ckpt['state_dict'])
print("Model Loaded")
frame = get_frame(args.method, model, args)
def run(args, kwargs):
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
args.snap_dir = snap_dir = \
'snapshots/discrete_logisticcifar10_flows_2_levels_3__2019-09-27_13_08_49/'
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
final_model = torch.load(snap_dir + 'a.model')
if hasattr(final_model, 'module'):
final_model = final_model.module
final_model = final_model.cuda()
sizes = []
errors = []
bpds = []
import time
start = time.time()
t = 0
with torch.no_grad():
for data, _ in test_loader:
if args.cuda:
data = data.cuda()
state, state_sizes, bpd, error = \
encode_images(data, final_model, decode=not args.no_decode)
errors += [error]
bpds.extend(bpd)
sizes.extend(state_sizes)
t += len(data)
print(
'Examples: {}/{} bpd compression: {:.3f} error: {},'
' analytical bpd {:.3f}'.format(
t, len(test_loader.dataset),
np.mean(sizes) / np.prod(data.size()[1:]),
np.sum(errors),
np.mean(bpds)
))
if args.no_decode:
print('Not testing decoding.')
else:
print('Error: {}'.format(np.sum(errors)))
print('Took {:.3f} seconds / example'.format((time.time() - start) / t))
print('Final bpd: {:.3f} error: {}'.format(
np.mean(sizes) / np.prod(data.size()[1:]),
np.sum(errors)))
from aae import AdversarialAutoencoder
from utils.configuration import Config
import numpy as np
from utils.load_data import load_dataset
#os.environ["CUDA_VISIBLE_DEVICES"]="1"
np.random.seed(100)
#tf.set_random_seed(0)
# == CONFIGURATION ==
config = Config()
# == DATASET ==
x_train, x_val, x_test, y_train, y_val, y_test = load_dataset(
config.dataset_name)
aae_model = AdversarialAutoencoder(config)
aae_model.vae.load_weights(str(config.latent_dim) + "ave_weights.hdf5")
# == TRAINING ==
if config.dataset_name == 'mnistDynamic':
print('***training with dynamic binarization***')
#使用测试集生成一些图片对比一下
batch_id = 5
x_sample = x_test[batch_id * config.batch_size:config.batch_size *
(batch_id + 1), :]
p_z_y = np.array([[0.]] * config.batch_size, dtype=np.float32)
x_reconstruct = aae_model.vae.predict([p_z_y, p_z_y, x_sample],
batch_size=config.batch_size)
dim=1,
keepdim=True) # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
total_sample += data.size()[0]
if batch_idx > 10: break
log_message = '\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss / (batch_idx + 1), correct, total_sample,
100. * correct / total_sample)
logger.info(log_message)
model.train()
if __name__ == '__main__':
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
prior_clamp = args.vae_clamp
args.vae_clamp = 100.
model = VAE.HouseholderSylvesterVAE(args).to(device)
frame = get_frame(args.method, model, args)
logger.info(model)
#logger.info("Number of trainable parameters: {}".format(count_parameters(model)))
if args.optim == 'SGD':
optimizer = getattr(optim, args.optim)(model.parameters(),
lr=args.warm_lr,
weight_decay=args.weight_decay,
momentum=args.momentum)
elif args.optim == 'Adamax':
datapath = 'omniglot_char_files/'
arguments.dynamic_binarization = 0
arguments.input_type = 'binary'
elif arguments.dataset_name == 'fashion_mnist':
Lclass = 10
datapath = 'fashion_mnist_files/'
arguments.dynamic_binarization = 0
arguments.input_type = 'gray'
elif arguments.dataset_name == 'mnist_plus_fmnist':
Lclass = 20
datapath = 'mnist_plus_fmnist_files/'
arguments.dynamic_binarization = 0
arguments.input_type = 'binary'
if not os.path.exists(datapath):
train_loader, val_loader, test_loader, arguments = load_dataset(arguments)
#sanity check for omniglot dataset
if arguments.dataset_name == 'omniglot_char':
tr_set = train_loader.dataset.__dict__['tensors'][0][:2500].reshape(
-1, 1, 28, 28)
tr_set_labels = train_loader.dataset.__dict__['tensors'][1]
val_set_labels = val_loader.dataset.__dict__['tensors'][1]
num_labels = np.zeros(max(tr_set_labels))
for k in range(max(tr_set_labels)):
num_labels[k] = (val_set_labels == k).sum().item()
assert min(num_labels) > 0, 'validation set has empty classes!'
ut.separate_datasets(train_loader, 'train', Lclass, datapath)
def run(args, kwargs):
args.model_signature = str(datetime.datetime.now())[0:19]
model_name = args.dataset_name + '_' + args.model_name + '_' + args.prior + '(M_' + str(
args.M) + ')' + '(F_' + str(args.F) + ')' + '_wu(' + str(
args.warmup) + ')' + '_z1_' + str(args.z1_size) + '_hidden_' + str(
args.number_hidden) + '_ksi_' + str(args.ksi)
if args.FI is True:
model_name += '_FI'
else:
args.F = 0
if args.MI is True:
model_name += '_MI'
else:
args.M = 0
# DIRECTORY FOR SAVING
#snapshots_path = 'snapshots/'
snapshots_path = args.snapshot_dir
if not os.path.exists(snapshots_path):
os.makedirs(snapshots_path)
dir = snapshots_path + args.model_signature + '_' + model_name + '/'
if not os.path.exists(dir):
os.makedirs(dir)
# LOAD DATA=========================================================================================================
print('load data')
# loading data
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# CREATE MODEL======================================================================================================
print('create model')
if args.dataset_name == "celeba":
args.model_name = "celebavae"
# importing model
if args.model_name == 'vae':
from models.VAE import VAE
elif args.model_name == 'hvae_2level':
from models.HVAE_2level import VAE
elif args.model_name == 'convhvae_2level':
from models.convHVAE_2level import VAE
elif args.model_name == 'convvae':
from models.convVAE import VAE
elif args.model_name == 'pixelhvae_2level':
from models.PixelHVAE_2level import VAE
elif args.model_name == 'pixelvae':
from models.pixelVAE import VAE
elif args.model_name == 'iaf_vae':
from models.VAE_ccLinIAF import VAE
elif args.model_name == 'rev_vae':
from models.REV_VAE import VAE
elif args.model_name == 'rev_pixelvae':
from models.REV_pixelVAE import VAE
elif args.model_name == 'celebavae':
from models.CelebaVAE import VAE
else:
raise Exception('Wrong name of the model!')
model = VAE(args)
if args.cuda:
model.cuda()
optimizer = AdamNormGrad(model.parameters(), lr=args.lr)
# ======================================================================================================================
print(args)
with open(dir + 'vae_experiment_log.txt', 'a') as f:
print(args, file=f)
# ======================================================================================================================
print('perform experiment')
from utils.perform_experiment import experiment_vae
experiment_vae(args,
train_loader,
val_loader,
test_loader,
model,
optimizer,
dir,
model_name=args.model_name)
# ======================================================================================================================
print(
'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-'
)
with open(dir + '/vae_experiment_log.txt', 'a') as f:
print(
'-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n',
file=f)
def run():
model_name = args.model_name
if model_name == 'vae_HF':
args.number_combination = 0
elif model_name == 'vae_ccLinIAF':
args.number_of_flows = 1
if args.model_name == 'vae_HF':
model_name = model_name + '(T_' + str(args.number_of_flows) + ')'
elif args.model_name == 'vae_ccLinIAF':
model_name = model_name + '(K_' + str(args.number_combination) + ')'
model_name = model_name + '_wu(' + str(args.warmup) + ')' + '_z1_' + str(args.z1_size)
if args.z2_size > 0:
model_name = model_name + '_z2_' + str(args.z2_size)
print(args)
with open('vae_experiment_log.txt', 'a') as f:
print(args, file=f)
# DIRECTORY FOR SAVING
snapshots_path = 'snapshots/'
dir = snapshots_path + model_name + '/'
if not os.path.exists(dir):
os.makedirs(dir)
# LOAD DATA=========================================================================================================
print('load data')
if args.dataset_name == 'dynamic_mnist':
args.dynamic_binarization = True
else:
args.dynamic_binarization = False
# loading data
train_loader, val_loader, test_loader = load_dataset(args)
# CREATE MODEL======================================================================================================
print('create model')
# importing model
if args.model_name == 'vae':
from models.VAE import VAE
elif args.model_name == 'vae_HF':
from models.VAE_HF import VAE
elif args.model_name == 'vae_ccLinIAF':
from models.VAE_ccLinIAF import VAE
else:
raise Exception('Wrong name of the model!')
model = VAE(args)
if args.cuda:
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ======================================================================================================================
print('perform experiment')
from utils.perform_experiment import experiment_vae
experiment_vae(args, train_loader, val_loader, test_loader, model, optimizer, dir, model_name = args.model_name)
# ======================================================================================================================
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-')
with open('vae_experiment_log.txt', 'a') as f:
print('-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n', file=f)
def run(args, kwargs):
# Would probably help, but experiments were done before.
args.grad_norm_enabled = False
print('\nMODEL SETTINGS: \n', args, '\n')
print("Random Seed: ", args.manual_seed)
if 'imagenet' in args.dataset and args.evaluate_interval_epochs > 5:
args.evaluate_interval_epochs = 5
# ==================================================================================================================
# SNAPSHOTS
# ==================================================================================================================
args.model_signature = str(datetime.datetime.now())[0:16].replace(' ', '_')
args.model_signature = args.model_signature.replace(':', '_')
snapshots_path = args.out_dir
snap_dir = snapshots_path + '/'
snap_dir += args.exp_name + args.dataset + '_' + 'flows_' + str(
args.n_subflows)
snap_dir = snap_dir + '_' + args.model_signature + '/'
args.snap_dir = snap_dir
if not os.path.exists(snap_dir):
os.makedirs(snap_dir)
with open(snap_dir + 'log.txt', 'a') as ff:
print('\nMODEL SETTINGS: \n', args, '\n', file=ff)
writer = tensorboardX.SummaryWriter(logdir=snap_dir)
# SAVING
torch.save(args, snap_dir + '.config')
# ==================================================================================================================
# LOAD DATA
# ==================================================================================================================
train_loader, val_loader, test_loader, args = load_dataset(args, **kwargs)
# ==================================================================================================================
# SELECT MODEL
# ==================================================================================================================
# flow parameters and architecture choice are passed on to model through
# args
model_pv = Flow(args,
args.input_size,
n_levels=args.n_levels,
n_subflows=args.n_subflows,
use_splitprior=args.use_splitprior,
n_context=None,
normalize_translation=128.,
normalize_scale=256.)
if args.dequantize_distribution == 'uniform':
model_hx = None
model_qu_x = Uniform(args.input_size)
elif args.dequantize_distribution == 'flow':
model_hx = torch.nn.Sequential(
Normalize_without_ldj(translation=128., scale=256.),
DenseNet(args,
input_size=(3, args.input_size[1], args.input_size[2]),
n_inputs=3,
n_outputs=args.n_context,
depth=4,
growth=32,
dropout_p=args.dropout_p),
torch.nn.Conv2d(args.n_context,
args.n_context,
kernel_size=2,
stride=2,
padding=0),
DenseNet(args,
n_inputs=args.n_context,
input_size=(3, args.input_size[1], args.input_size[2]),
n_outputs=args.n_context,
depth=4,
growth=32,
dropout_p=args.dropout_p),
)
model_qu_x = TemplateDistribution(
transformations=[ReverseTransformation(Sigmoid())],
distribution=Flow(args,
args.input_size,
n_levels=args.dequantize_levels,
n_subflows=args.dequantize_subflows,
n_context=args.n_context,
use_splitprior=False,
normalize_translation=0.,
normalize_scale=1.,
parametrize_inverse=True))
else:
raise ValueError
model = DiscreteLowerboundModel(model_pv, model_qu_x, model_hx)
args.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
model.to(args.device)
model_sample = model
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = torch.nn.DataParallel(model, dim=0)
def lr_lambda(epoch):
factor = min(1., (epoch + 1) / args.warmup) * np.power(
args.lr_decay, epoch)
print('Learning rate factor:', factor)
return factor
optimizer = optim.Adamax(model.parameters(),
lr=args.learning_rate,
eps=1.e-7)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
lr_lambda,
last_epoch=-1)
# Log the number of params.
number_of_params = np.sum(
[np.prod(tensor.size()) for tensor in model.parameters()])
fn = snap_dir + 'log.txt'
with open(fn, 'a') as ff:
msg = 'Number of Parameters: {}'.format(number_of_params)
print(msg, file=ff)
print(msg)
# ==================================================================================================================
# TRAINING
# ==================================================================================================================
train_bpd = []
val_bpd = []
# for early stopping
best_val_bpd = np.inf
best_train_bpd = np.inf
epoch = 0
train_times = []
model.eval()
model.train()
for epoch in range(1, args.epochs + 1):
t_start = time.time()
tr_bpd = train(epoch, train_loader, model, optimizer, args)
scheduler.step()
train_bpd.append(tr_bpd)
writer.add_scalar('train bpd', train_bpd[-1], epoch)
train_times.append(time.time() - t_start)
print('One training epoch took %.2f seconds' % (time.time() - t_start))
if epoch < 25 or epoch % args.evaluate_interval_epochs == 0:
tr_bpd = evaluate(train_loader, model, args, iw_samples=1)
v_bpd = evaluate(val_loader, model, args, iw_samples=1)
# Logging message.
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttrain bpd {:.3f}\tval bpd {:.3f}\t'.format(
epoch, tr_bpd, v_bpd)
print(msg, file=ff)
# Sample and time sampling.
torch.cuda.synchronize()
start_sample = time.time()
plot_samples(model_sample, args, epoch, v_bpd)
torch.cuda.synchronize()
print('Sampling took {} seconds'.format(time.time() -
start_sample))
val_bpd.append(v_bpd)
writer.add_scalar('val bpd', v_bpd, epoch)
# Model save based on val performance
if v_bpd < best_val_bpd:
best_train_bpd = tr_bpd
best_val_bpd = v_bpd
try:
if hasattr(model, 'module'):
torch.save(model.module, snap_dir + 'a.model')
else:
torch.save(model, snap_dir + 'a.model')
torch.save(optimizer, snap_dir + 'a.optimizer')
print('->model saved<-')
except:
print('Saving was unsuccessful.')
print('(BEST: train bpd {:.4f}, val bpd {:.4f})\n'.format(
best_train_bpd, best_val_bpd))
if math.isnan(v_bpd):
raise ValueError('NaN encountered!')
# training time per epoch
train_times = np.array(train_times)
mean_train_time = np.mean(train_times)
std_train_time = np.std(train_times, ddof=1)
print('Average train time per epoch: {:.2f} +/- {:.2f}'.format(
mean_train_time, std_train_time))
# ========================================================================
# EVALUATION
# ========================================================================
final_model = torch.load(snap_dir + 'a.model')
test_bpd = evaluate(test_loader, final_model, args)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttest negative elbo bpd {:.4f}'.format(epoch, test_bpd)
print(msg, file=ff)
test_bpd = evaluate(test_loader, final_model, args, iw_samples=1000)
with open(snap_dir + 'log.txt', 'a') as ff:
msg = 'epoch {}\ttest negative log_px bpd {:.4f}'.format(
epoch, test_bpd)
print(msg, file=ff)
