def __init__(self):
self.serial_manager = SerialManager()
try:
self.database = Database()
except KeyError:
print('Error loading database')
self.abort_test()
self.csv_logger = CSVLogger()
def reaching_tasks(model, position_import_model_id=None):
ids = [model + "reachclose", model + 'reachfar']
positions = np.zeros((2, 40, 2))
if position_import_model_id is None:
positions[0] = np.random.uniform(low=1, high=8, size=(40, 2)) * (
-2 * np.random.randint(low=0, high=2, size=(40, 2)) + 1)
positions[1] = np.random.uniform(low=8, high=18, size=(40, 2)) * (
-2 * np.random.randint(low=0, high=2, size=(40, 2)) + 1)
else:
positions[0] = CSVLogger.import_log(
log_id=position_import_model_id + "reach" + "close" + "0n",
n=40,
length=5000,
columns=["rubb_Shoulder", "rubb_Elbow"])[:, 0, :]
positions[1] = CSVLogger.import_log(
log_id=position_import_model_id + "reach" + "far" + "0n",
n=40,
length=5000,
columns=["rubb_Shoulder", "rubb_Elbow"])[:, 0, :]
n_iterations = 5000
for i in range(0, 40):
for c in range(positions.shape[0]):
unity = UnityContainer(editor_mode)
unity.initialise_environment()
visual_decoder = VAE_CNN()
visual_decoder.load_from_file(model_id)
unity.set_visible_arm(VisibleArm.RealArm)
unity.reset()
unity.set_rubber_arm_rotation(np.array([[-17, -10]]))
# Set real arm to rubber arm position to get attr image
unity.set_rotation(unity.get_rubber_joint_observation())
attr_image = np.squeeze(unity.get_visual_observation())
unity.set_rotation(np.zeros((1, 2)))
data_range = np.load(visual_decoder.SAVE_PATH + "/" + model_id +
"/data_range" + model_id + ".npy")
central = FepAgent(unity,
visual_decoder,
data_range,
enable_action=True,
attractor_image=attr_image)
log_id = ids[c] + str(central.sp_noise_variance) + "n" + str(i)
central.run_simulation(log_id, n_iterations)
unity.close()
def __init__(self):
self.serial_manager = SerialManager()
try:
self.database = Database()
except KeyError:
print(
'Error loading database. Did you set the TEST_STAND_DB '
'environment variable to the name of your InfluxDB database?')
self.abort_test()
self.csv_logger = CSVLogger()
# thread for interpreting incoming serial data
self.telemetry_thread = threading.Thread(target=self.telemetry_loop)
self.thread_is_running = False
def main():
display = Display(config.display_device, config.logger)
weather = WeatherSensor(config.weather_sensor)
sqm_reader = SQM(config)
csv = CSVLogger(config.csv_logfile)
last_weather_read = 0
last_sqm_read = 0
weather_data, sqm_data, sqm_frequency = None, None, None
while True:
now = time.time()
if now - last_weather_read > config.read_weather_every:
last_weather_read = now
weather_data = weather.read()
if now - last_sqm_read > config.read_sqm_every:
display.clear()
try:
sqm_data, sqm_frequency = sqm_reader.read_median_sqm(readings=config.sqm_readings)
except:
config.logger.error('Error retrieving sqm data: ', exc_info=True)
sqm_data, sqm_frequency = None, None
last_sqm_read = now
csv.line(weather_data['temp_degrees'], weather_data['humidity'], weather_data['hPa'], sqm_data, sqm_frequency)
display_brightness = 0
if sqm_data and sqm_data < 8:
display_brightness = 128
if sqm_data and sqm_data < 6.5:
display_brightness = 255
display.set_weather(weather_data, render=False)
display.set_sqm(sqm_data, render=False)
display.render(contrast=display_brightness)
time.sleep(1)
def main():
"""
Initialise SolarInfo, Aircon and CSVLogger instances with all settings for
target grid import (or export) range, read frequency, aircon set frequency,
parameters logged to CSV, sleep mode settings and thresholds.
Eventually, all this should be read a config file and even be settable via a
web API.
"""
logging.basicConfig(level=logging.INFO)
#logging.getLogger('daikin_api').setLevel(logging.DEBUG)
#logging.getLogger('solaredge_api').setLevel(logging.DEBUG)
logging.getLogger('solaredge_modbus_tcp').setLevel(logging.DEBUG)
config_file = 'config.yaml'
yaml = YAML()
with open(config_file, 'r') as stream:
config = yaml.load(stream)
# Ensure A/C sleep mode settings are strings
for key, value in config['sleep_mode_settings'].items():
config['sleep_mode_settings'][key] = str(value)
aircons = [Aircon(ac['number'], ac['url']) for ac in config['aircons']]
csv_logger = None
if config['csv_file']:
csv_logger = CSVLogger(config['csv_file'], config['csv_headers'])
recent_values = None
if config['recent_values'] and config['recent_values']['max_size']:
recent_values = {
'headers': None,
'values': deque(maxlen=config['recent_values']['max_size'])
}
bbc = BrickBatteryCharger(config,
aircons,
SolarInfo(config['inverter_host']),
server=BrickBatteryHTTPServer(
config['listen']['interface'],
config['listen']['port'], config_file),
csv=csv_logger,
recent_values=recent_values)
bbc.charge()
def run_simulation(self, log_id: str, n_iterations: int):
"""
Run a simulation by iteratively performing updating steps
:param log_id: file identifier that will be used to write operation_logs to.
If not specified, no log will be created
:param n_iterations: number of iteration to run the simulation for
"""
if log_id is not None:
csv_logger = CSVLogger(log_id)
csv_logger.write_header(self)
# Initialise live plot
plot = LivePlot(3, 2)
for i in range(n_iterations):
self.s_p = add_gaussian_noise(self.env.get_joint_observation(), 0, self.sp_noise_variance)
self.s_v[0, 0] = np.squeeze(self.env.get_visual_observation())
self.gamma = self.get_posterior_gamma(self.env.get_touch_observation())
self.gamma_tracker.append(self.gamma)
self.active_inference_step()
if i % self.plot_interval == 0:
plot.update_live_plot(self)
if i % 10 == 0:
print("Iteration", i, "action:", self.a, "belief", self.mu, "GT", self.s_p, "Ev attr",
self.attr_error_tracker)
if log_id is not None:
csv_logger.write_iteration(self, i)
if self.action_enabled:
self.env.act(self.a)
else:
self.env.act(np.zeros((1, 2)))
plt.close()
'z_mean', 'z_std'
]
svd_fieldnames = [
'global_iteration', 'mean_recons_error', 'condition_num', 'max_sv',
'min_sv', 'inverse_condition_num', 'inverse_max_sv', 'inverse_min_sv'
]
epoch_fieldnames = [
'epoch', 'time_elapsed', 'train_loss', 'test_loss', 'train_acc',
'test_acc', 'mean_train_recons_error', 'std_train_recons_error',
'mean_test_recons_error', 'std_test_recons_error'
]
iteration_logger = CSVLogger(fieldnames=iteration_fieldnames,
filename=os.path.join(save_dir,
'iteration_log.csv'))
every_N_logger = CSVLogger(fieldnames=every_N_fieldnames,
filename=os.path.join(save_dir, 'every_N_log.csv'))
svd_logger = CSVLogger(fieldnames=svd_fieldnames,
filename=os.path.join(save_dir, 'svd_log.csv'))
epoch_logger = CSVLogger(fieldnames=epoch_fieldnames,
filename=os.path.join(save_dir, 'epoch_log.csv'))
# Data loading
# ------------
cifar_mean = [x / 255.0 for x in [125.3, 123.0, 113.9]]
cifar_std = [x / 255.0 for x in [63.0, 62.1, 66.7]]
tf = transforms.Compose([
transforms.RandomCrop(32, padding=4),
from sklearn.metrics import accuracy_score, r2_score, mean_squared_error
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import OneHotEncoder
# from lime.lime_tabular import LimeTabularExplainer
from csv_logger import CSVLogger
from lime_tabular import LimeTabularExplainer
from models import NN_with_EntityEmbedding, NN
from data_script import get_data
run_id = 0
log_file = '/content/drive/My Drive/myenv/log' + str(run_id)
logger = CSVLogger(name='mylogger', log_file=log_file, level='info')
filtered_tasks_ids = [
3, 20, 24, 41, 45, 49, 3492, 3493, 3494, 3560, 34537, 34539, 146195
]
epochs = 50
batch_size = 128
for j in range(len(filtered_tasks_ids)):
print("++++++++++++++++++++++++++++++++++++++++++++++++")
print("++++++++++ tasks: " + str(j + 1) + "/" +
str(len(filtered_tasks_ids)) + "+++")
print("++++++++++++++++++++++++++++++++++++++++++++++++")
# within different runs, split traning/testing sets with different
# random_state.
taskid = filtered_tasks_ids[j]
X, X_train, X_test, y_train_str, y_test_str, y_train_int, y_test_int, feature_names, class_names, categorical_names, categorical_features = get_data(
taskid, random_state=run_id)
# nn with embedding related
]
# j is the index of filtered_tasks_ids
j = 0
# Google Colab
log_file = '/content/drive/My Drive/myenv/logs/ae_mlp_coefs_stability_' + str(
j)
blackbox_path = '/content/drive/My Drive/myenv/trained_models_with_entity_embedding/'
embedding_path = '/content/drive/My Drive/myenv/trained_models_with_mlp_embedding/'
# # Local Host
# log_file = 'logs/try_mlp_embedding_local_fidelity_'+str(j)
# blackbox_path = 'trained_models_with_entity_embedding/'
# embedding_path = 'trained_models_with_mlp_embedding/'
logger = CSVLogger(name='mylogger', log_file=log_file, level='info')
num_samples_iter = list(range(500, 5500, 500))
epochs = 50
# epochs = 1
batch_size = 128
num_hidden = 500
# num_hidden = 10
explain_index = 0
repeat_times = 10
# repeat_times = 1
# change !
# blackbox = 'rf'
blackbox = 'nn'
embedding_type = 'mlp' # 'entity'
# Google Colab log_file = '/content/drive/My Drive/myenv/logs/local_fidelity_' + str(j) blackbox_path = '/content/drive/My Drive/myenv/trained_models_with_entity_embedding/' # embedding_path = '/content/drive/My Drive/myenv/trained_models_with_mlp_embedding/' # # Baidu AI Studio # log_file = '' # model_path = '' # # Local Host # log_file = 'logs/try_mlp_embedding_local_fidelity_'+str(j) # blackbox_path = 'trained_models_with_entity_embedding/' # embedding_path = 'trained_models_with_mlp_embedding/' logger = CSVLogger(name='mylogger', log_file=log_file, level='info') num_samples_iter = list(range(500, 5500, 500)) epochs = 50 # epochs = 1 batch_size = 128 num_hidden = 500 # num_hidden = 10 # blackbox = 'rf' blackbox = 'nn' # embedding_type = 'mlp' # 'entity' embedding_type = 'entity' strategy_list, taskid_list, runid_list, num_samples_list, score_r2_list, score_mse_list, score_mae_list, score_evc_list = [], [], [], [], [], [], [], [] # Experiment 1: local fidelity
type=str,
default='finetuned_checkpoints',
help='Save directory for the fine-tuned checkpoint')
args = parser.parse_args()
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
test_id = '{}_{}_{}_lr{}_wd{}_aug{}'.format(args.dataset, args.model,
args.optimizer, args.lr,
args.wdecay,
int(args.data_augmentation))
filename = os.path.join(args.save_dir, test_id + '.csv')
csv_logger = CSVLogger(fieldnames=[
'epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'test_loss',
'test_acc'
],
filename=filename)
model = torch.load(args.load_checkpoint)
# TODO(PV): Load saved optimizer from the training run
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
nesterov=True,
weight_decay=args.wdecay)
# Set random regularization hyperparameters
# data_augmentation_hparams = {} # Random values for hue, saturation, brightness, contrast, rotation, etc.
if args.dataset == 'cifar10':
def main(args):
#
use_cuda = not args.no_cuda and torch.cuda.is_available()
set_random_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
if args.dataset == 'mnist':
train_data = get_dataset('mnist-train', args.dataroot)
test_data = get_dataset('mnist-test', args.dataroot)
train_tr = test_tr = get_transform('mnist_normalize')
if args.dataset == 'cifar10':
train_tr_name = 'cifar_augment_normalize' if args.data_augmentation else 'cifar_normalize'
train_data = get_dataset('cifar10-train', args.dataroot)
test_data = get_dataset('cifar10-test', args.dataroot)
train_tr = get_transform(train_tr_name)
test_tr = get_transform('cifar_normalize')
if args.dataset == 'cifar-fs-train':
train_tr_name = 'cifar_augment_normalize' if args.data_augmentation else 'cifar_normalize'
train_data = get_dataset('cifar-fs-train-train', args.dataroot)
test_data = get_dataset('cifar-fs-train-test', args.dataroot)
train_tr = get_transform(train_tr_name)
test_tr = get_transform('cifar_normalize')
if args.dataset == 'miniimagenet':
train_data = get_dataset('miniimagenet-train-train', args.dataroot)
test_data = get_dataset('miniimagenet-train-test', args.dataroot)
train_tr = get_transform('cifar_augment_normalize_84' if args.data_augmentation else 'cifar_normalize')
test_tr = get_transform('cifar_normalize')
model = ResNetClassifier(train_data['n_classes'], train_data['im_size']).to(device)
if args.ckpt_path != '':
loaded = torch.load(args.ckpt_path)
model.load_state_dict(loaded)
ipdb.set_trace()
if args.eval:
acc = test(args, model, device, test_loader, args.n_eval_batches)
print("Eval Acc: ", acc)
sys.exit()
# Trace logging
mkdir(args.output_dir)
eval_fieldnames = ['global_iteration','val_acc','train_acc']
eval_logger = CSVLogger(every=1,
fieldnames=eval_fieldnames,
resume=args.resume,
filename=os.path.join(args.output_dir, 'eval_log.csv'))
wandb.run.name = os.path.basename(args.output_dir)
wandb.run.save()
wandb.watch(model)
if args.optim == 'adadelta':
optimizer = optim.Adadelta(model.parameters(), lr=args.lr)
elif args.optim == 'adam':
optimizer = optim.Adam(model.parameters(), lr=args.lr)
elif args.optim == 'sgd':
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=0.9, nesterov=True, weight_decay=5e-4)
if args.dataset == 'mnist':
scheduler = StepLR(optimizer, step_size=1, gamma=.7)
else:
scheduler = MultiStepLR(optimizer, milestones=[60, 120, 160], gamma=0.2)
start_epoch = 1
if args.resume:
last_ckpt_path = os.path.join(args.output_dir, 'last_ckpt.pt')
if os.path.exists(last_ckpt_path):
loaded = torch.load(last_ckpt_path)
model.load_state_dict(loaded['model_sd'])
optimizer.load_state_dict(loaded['optimizer_sd'])
scheduler.load_state_dict(loaded['scheduler_sd'])
start_epoch = loaded['epoch']
# It's important to set seed again before training b/c dataloading code
# might have reset the seed.
set_random_seed(args.seed)
best_val = 0
if args.db:
scheduler = MultiStepLR(optimizer, milestones=[1, 2, 3, 4], gamma=0.1)
args.epochs = 5
for epoch in range(start_epoch, args.epochs + 1):
if epoch % args.ckpt_every == 0:
torch.save(model.state_dict(), os.path.join(args.output_dir , f"ckpt_{epoch}.pt"))
stats_dict = {'global_iteration':epoch}
val = stats_dict['val_acc'] = test(args, model, device, test_data, test_tr, args.n_eval_batches)
stats_dict['train_acc'] = test(args, model, device, train_data, test_tr, args.n_eval_batches)
grid = make_grid(torch.stack([train_tr(x) for x in train_data['x'][:30]]), nrow=6).permute(1,2,0).numpy()
img_dict = {"examples": [wandb.Image(grid, caption="Data batch")]}
wandb.log(stats_dict)
wandb.log(img_dict)
eval_logger.writerow(stats_dict)
plot_csv(eval_logger.filename, os.path.join(args.output_dir, 'iteration_plots.png'))
train(args, model, device, train_data, train_tr, optimizer, epoch)
scheduler.step(epoch)
if val > best_val:
best_val = val
torch.save(model.state_dict(), os.path.join(args.output_dir , f"ckpt_best.pt"))
# For `resume`
model.cpu()
torch.save({
'model_sd': model.state_dict(),
'optimizer_sd': optimizer.state_dict(),
'scheduler_sd': scheduler.state_dict(),
'epoch': epoch + 1
}, os.path.join(args.output_dir, "last_ckpt.pt"))
model.to(device)
def main(dataset, dataroot, download, augment, batch_size, eval_batch_size,
epochs, saved_model, seed, hidden_channels, K, L, actnorm_scale,
flow_permutation, flow_coupling, LU_decomposed, learn_top,
y_condition, y_weight, max_grad_clip, max_grad_norm, lr, n_workers,
cuda, n_init_batches, warmup_steps, output_dir, saved_optimizer,
warmup, fresh, logittransform, gan, disc_lr, sn, flowgan, eval_every,
ld_on_samples, weight_gan, weight_prior, weight_logdet,
jac_reg_lambda, affine_eps, no_warm_up, optim_name, clamp, svd_every,
eval_only, no_actnorm, affine_scale_eps, actnorm_max_scale,
no_conv_actnorm, affine_max_scale, actnorm_eps, init_sample, no_split,
disc_arch, weight_entropy_reg, db):
check_manual_seed(seed)
ds = check_dataset(dataset, dataroot, augment, download)
image_shape, num_classes, train_dataset, test_dataset = ds
# Note: unsupported for now
multi_class = False
train_loader = data.DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers,
drop_last=True)
test_loader = data.DataLoader(test_dataset,
batch_size=eval_batch_size,
shuffle=False,
num_workers=n_workers,
drop_last=False)
model = Glow(image_shape, hidden_channels, K, L, actnorm_scale,
flow_permutation, flow_coupling, LU_decomposed, num_classes,
learn_top, y_condition, logittransform, sn, affine_eps,
no_actnorm, affine_scale_eps, actnorm_max_scale,
no_conv_actnorm, affine_max_scale, actnorm_eps, no_split)
model = model.to(device)
if disc_arch == 'mine':
discriminator = mine.Discriminator(image_shape[-1])
elif disc_arch == 'biggan':
discriminator = cgan_models.Discriminator(
image_channels=image_shape[-1], conditional_D=False)
elif disc_arch == 'dcgan':
discriminator = DCGANDiscriminator(image_shape[0], 64, image_shape[-1])
elif disc_arch == 'inv':
discriminator = InvDiscriminator(
image_shape, hidden_channels, K, L, actnorm_scale,
flow_permutation, flow_coupling, LU_decomposed, num_classes,
learn_top, y_condition, logittransform, sn, affine_eps, no_actnorm,
affine_scale_eps, actnorm_max_scale, no_conv_actnorm,
affine_max_scale, actnorm_eps, no_split)
discriminator = discriminator.to(device)
D_optimizer = optim.Adam(filter(lambda p: p.requires_grad,
discriminator.parameters()),
lr=disc_lr,
betas=(.5, .99),
weight_decay=0)
if optim_name == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=lr,
betas=(.5, .99),
weight_decay=0)
elif optim_name == 'adamax':
optimizer = optim.Adamax(model.parameters(), lr=lr, weight_decay=5e-5)
if not no_warm_up:
lr_lambda = lambda epoch: min(1.0, (epoch + 1) / warmup)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
lr_lambda=lr_lambda)
iteration_fieldnames = [
'global_iteration', 'fid', 'sample_pad', 'train_bpd', 'eval_bpd',
'pad', 'batch_real_acc', 'batch_fake_acc', 'batch_acc'
]
iteration_logger = CSVLogger(fieldnames=iteration_fieldnames,
filename=os.path.join(output_dir,
'iteration_log.csv'))
iteration_fieldnames = [
'global_iteration', 'condition_num', 'max_sv', 'min_sv',
'inverse_condition_num', 'inverse_max_sv', 'inverse_min_sv'
]
svd_logger = CSVLogger(fieldnames=iteration_fieldnames,
filename=os.path.join(output_dir, 'svd_log.csv'))
#
test_iter = test_loader.__iter__()
N_inception = 1000
x_real_inception = torch.cat([
test_iter.__next__()[0].to(device)
for _ in range(N_inception // args.batch_size + 1)
], 0)[:N_inception]
x_real_inception = x_real_inception + .5
x_for_recon = test_iter.__next__()[0].to(device)
def gan_step(engine, batch):
assert not y_condition
if 'iter_ind' in dir(engine):
engine.iter_ind += 1
else:
engine.iter_ind = -1
losses = {}
model.train()
discriminator.train()
x, y = batch
x = x.to(device)
def run_noised_disc(discriminator, x):
x = uniform_binning_correction(x)[0]
return discriminator(x)
real_acc = fake_acc = acc = 0
if weight_gan > 0:
fake = generate_from_noise(model, x.size(0), clamp=clamp)
D_real_scores = run_noised_disc(discriminator, x.detach())
D_fake_scores = run_noised_disc(discriminator, fake.detach())
ones_target = torch.ones((x.size(0), 1), device=x.device)
zeros_target = torch.zeros((x.size(0), 1), device=x.device)
D_real_accuracy = torch.sum(
torch.round(F.sigmoid(D_real_scores)) ==
ones_target).float() / ones_target.size(0)
D_fake_accuracy = torch.sum(
torch.round(F.sigmoid(D_fake_scores)) ==
zeros_target).float() / zeros_target.size(0)
D_real_loss = F.binary_cross_entropy_with_logits(
D_real_scores, ones_target)
D_fake_loss = F.binary_cross_entropy_with_logits(
D_fake_scores, zeros_target)
D_loss = (D_real_loss + D_fake_loss) / 2
gp = gradient_penalty(
x.detach(), fake.detach(),
lambda _x: run_noised_disc(discriminator, _x))
D_loss_plus_gp = D_loss + 10 * gp
D_optimizer.zero_grad()
D_loss_plus_gp.backward()
D_optimizer.step()
# Train generator
fake = generate_from_noise(model,
x.size(0),
clamp=clamp,
guard_nans=False)
G_loss = F.binary_cross_entropy_with_logits(
run_noised_disc(discriminator, fake),
torch.ones((x.size(0), 1), device=x.device))
# Trace
real_acc = D_real_accuracy.item()
fake_acc = D_fake_accuracy.item()
acc = .5 * (D_fake_accuracy.item() + D_real_accuracy.item())
z, nll, y_logits, (prior, logdet) = model.forward(x,
None,
return_details=True)
train_bpd = nll.mean().item()
loss = 0
if weight_gan > 0:
loss = loss + weight_gan * G_loss
if weight_prior > 0:
loss = loss + weight_prior * -prior.mean()
if weight_logdet > 0:
loss = loss + weight_logdet * -logdet.mean()
if weight_entropy_reg > 0:
_, _, _, (sample_prior,
sample_logdet) = model.forward(fake,
None,
return_details=True)
# notice this is actually "decreasing" sample likelihood.
loss = loss + weight_entropy_reg * (sample_prior.mean() +
sample_logdet.mean())
# Jac Reg
if jac_reg_lambda > 0:
# Sample
x_samples = generate_from_noise(model,
args.batch_size,
clamp=clamp).detach()
x_samples.requires_grad_()
z = model.forward(x_samples, None, return_details=True)[0]
other_zs = torch.cat([
split._last_z2.view(x.size(0), -1)
for split in model.flow.splits
], -1)
all_z = torch.cat([other_zs, z.view(x.size(0), -1)], -1)
sample_foward_jac = compute_jacobian_regularizer(x_samples,
all_z,
n_proj=1)
_, c2, h, w = model.prior_h.shape
c = c2 // 2
zshape = (batch_size, c, h, w)
randz = torch.randn(zshape).to(device)
randz = torch.autograd.Variable(randz, requires_grad=True)
images = model(z=randz,
y_onehot=None,
temperature=1,
reverse=True,
batch_size=0)
other_zs = [split._last_z2 for split in model.flow.splits]
all_z = [randz] + other_zs
sample_inverse_jac = compute_jacobian_regularizer_manyinputs(
all_z, images, n_proj=1)
# Data
x.requires_grad_()
z = model.forward(x, None, return_details=True)[0]
other_zs = torch.cat([
split._last_z2.view(x.size(0), -1)
for split in model.flow.splits
], -1)
all_z = torch.cat([other_zs, z.view(x.size(0), -1)], -1)
data_foward_jac = compute_jacobian_regularizer(x, all_z, n_proj=1)
_, c2, h, w = model.prior_h.shape
c = c2 // 2
zshape = (batch_size, c, h, w)
z.requires_grad_()
images = model(z=z,
y_onehot=None,
temperature=1,
reverse=True,
batch_size=0)
other_zs = [split._last_z2 for split in model.flow.splits]
all_z = [z] + other_zs
data_inverse_jac = compute_jacobian_regularizer_manyinputs(
all_z, images, n_proj=1)
# loss = loss + jac_reg_lambda * (sample_foward_jac + sample_inverse_jac )
loss = loss + jac_reg_lambda * (sample_foward_jac +
sample_inverse_jac +
data_foward_jac + data_inverse_jac)
if not eval_only:
optimizer.zero_grad()
loss.backward()
if not db:
assert max_grad_clip == max_grad_norm == 0
if max_grad_clip > 0:
torch.nn.utils.clip_grad_value_(model.parameters(),
max_grad_clip)
if max_grad_norm > 0:
torch.nn.utils.clip_grad_norm_(model.parameters(),
max_grad_norm)
# Replace NaN gradient with 0
for p in model.parameters():
if p.requires_grad and p.grad is not None:
g = p.grad.data
g[g != g] = 0
optimizer.step()
if engine.iter_ind % 100 == 0:
with torch.no_grad():
fake = generate_from_noise(model, x.size(0), clamp=clamp)
z = model.forward(fake, None, return_details=True)[0]
print("Z max min")
print(z.max().item(), z.min().item())
if (fake != fake).float().sum() > 0:
title = 'NaNs'
else:
title = "Good"
grid = make_grid((postprocess(fake.detach().cpu(), dataset)[:30]),
nrow=6).permute(1, 2, 0)
plt.figure(figsize=(10, 10))
plt.imshow(grid)
plt.axis('off')
plt.title(title)
plt.savefig(
os.path.join(output_dir, f'sample_{engine.iter_ind}.png'))
if engine.iter_ind % eval_every == 0:
def check_all_zero_except_leading(x):
return x % 10**np.floor(np.log10(x)) == 0
if engine.iter_ind == 0 or check_all_zero_except_leading(
engine.iter_ind):
torch.save(
model.state_dict(),
os.path.join(output_dir, f'ckpt_sd_{engine.iter_ind}.pt'))
model.eval()
with torch.no_grad():
# Plot recon
fpath = os.path.join(output_dir, '_recon',
f'recon_{engine.iter_ind}.png')
sample_pad = run_recon_evolution(
model,
generate_from_noise(model, args.batch_size,
clamp=clamp).detach(), fpath)
print(
f"Iter: {engine.iter_ind}, Recon Sample PAD: {sample_pad}")
pad = run_recon_evolution(model, x_for_recon, fpath)
print(f"Iter: {engine.iter_ind}, Recon PAD: {pad}")
pad = pad.item()
sample_pad = sample_pad.item()
# Inception score
sample = torch.cat([
generate_from_noise(model, args.batch_size, clamp=clamp)
for _ in range(N_inception // args.batch_size + 1)
], 0)[:N_inception]
sample = sample + .5
if (sample != sample).float().sum() > 0:
print("Sample NaNs")
raise
else:
fid = run_fid(x_real_inception.clamp_(0, 1),
sample.clamp_(0, 1))
print(f'fid: {fid}, global_iter: {engine.iter_ind}')
# Eval BPD
eval_bpd = np.mean([
model.forward(x.to(device), None,
return_details=True)[1].mean().item()
for x, _ in test_loader
])
stats_dict = {
'global_iteration': engine.iter_ind,
'fid': fid,
'train_bpd': train_bpd,
'pad': pad,
'eval_bpd': eval_bpd,
'sample_pad': sample_pad,
'batch_real_acc': real_acc,
'batch_fake_acc': fake_acc,
'batch_acc': acc
}
iteration_logger.writerow(stats_dict)
plot_csv(iteration_logger.filename)
model.train()
if engine.iter_ind + 2 % svd_every == 0:
model.eval()
svd_dict = {}
ret = utils.computeSVDjacobian(x_for_recon, model)
D_for, D_inv = ret['D_for'], ret['D_inv']
cn = float(D_for.max() / D_for.min())
cn_inv = float(D_inv.max() / D_inv.min())
svd_dict['global_iteration'] = engine.iter_ind
svd_dict['condition_num'] = cn
svd_dict['max_sv'] = float(D_for.max())
svd_dict['min_sv'] = float(D_for.min())
svd_dict['inverse_condition_num'] = cn_inv
svd_dict['inverse_max_sv'] = float(D_inv.max())
svd_dict['inverse_min_sv'] = float(D_inv.min())
svd_logger.writerow(svd_dict)
# plot_utils.plot_stability_stats(output_dir)
# plot_utils.plot_individual_figures(output_dir, 'svd_log.csv')
model.train()
if eval_only:
sys.exit()
# Dummy
losses['total_loss'] = torch.mean(nll).item()
return losses
def eval_step(engine, batch):
model.eval()
x, y = batch
x = x.to(device)
with torch.no_grad():
if y_condition:
y = y.to(device)
z, nll, y_logits = model(x, y)
losses = compute_loss_y(nll,
y_logits,
y_weight,
y,
multi_class,
reduction='none')
else:
z, nll, y_logits = model(x, None)
losses = compute_loss(nll, reduction='none')
return losses
trainer = Engine(gan_step)
# else:
# trainer = Engine(step)
checkpoint_handler = ModelCheckpoint(output_dir,
'glow',
save_interval=5,
n_saved=1,
require_empty=False)
trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {
'model': model,
'optimizer': optimizer
})
monitoring_metrics = ['total_loss']
RunningAverage(output_transform=lambda x: x['total_loss']).attach(
trainer, 'total_loss')
evaluator = Engine(eval_step)
# Note: replace by https://github.com/pytorch/ignite/pull/524 when released
Loss(lambda x, y: torch.mean(x),
output_transform=lambda x:
(x['total_loss'], torch.empty(x['total_loss'].shape[0]))).attach(
evaluator, 'total_loss')
if y_condition:
monitoring_metrics.extend(['nll'])
RunningAverage(output_transform=lambda x: x['nll']).attach(
trainer, 'nll')
# Note: replace by https://github.com/pytorch/ignite/pull/524 when released
Loss(lambda x, y: torch.mean(x),
output_transform=lambda x:
(x['nll'], torch.empty(x['nll'].shape[0]))).attach(
evaluator, 'nll')
pbar = ProgressBar()
pbar.attach(trainer, metric_names=monitoring_metrics)
# load pre-trained model if given
if saved_model:
print("Loading...")
print(saved_model)
loaded = torch.load(saved_model)
# if 'Glow' in str(type(loaded)):
# model = loaded
# else:
# raise
# # if 'Glow' in str(type(loaded)):
# # loaded = loaded.state_dict()
model.load_state_dict(loaded)
model.set_actnorm_init()
if saved_optimizer:
optimizer.load_state_dict(torch.load(saved_optimizer))
file_name, ext = os.path.splitext(saved_model)
resume_epoch = int(file_name.split('_')[-1])
@trainer.on(Events.STARTED)
def resume_training(engine):
engine.state.epoch = resume_epoch
engine.state.iteration = resume_epoch * len(
engine.state.dataloader)
@trainer.on(Events.STARTED)
def init(engine):
if saved_model:
return
model.train()
print("Initializing Actnorm...")
init_batches = []
init_targets = []
if n_init_batches == 0:
model.set_actnorm_init()
return
with torch.no_grad():
if init_sample:
generate_from_noise(model,
args.batch_size * args.n_init_batches)
else:
for batch, target in islice(train_loader, None,
n_init_batches):
init_batches.append(batch)
init_targets.append(target)
init_batches = torch.cat(init_batches).to(device)
assert init_batches.shape[0] == n_init_batches * batch_size
if y_condition:
init_targets = torch.cat(init_targets).to(device)
else:
init_targets = None
model(init_batches, init_targets)
@trainer.on(Events.EPOCH_COMPLETED)
def evaluate(engine):
evaluator.run(test_loader)
if not no_warm_up:
scheduler.step()
metrics = evaluator.state.metrics
losses = ', '.join(
[f"{key}: {value:.2f}" for key, value in metrics.items()])
print(f'Validation Results - Epoch: {engine.state.epoch} {losses}')
timer = Timer(average=True)
timer.attach(trainer,
start=Events.EPOCH_STARTED,
resume=Events.ITERATION_STARTED,
pause=Events.ITERATION_COMPLETED,
step=Events.ITERATION_COMPLETED)
@trainer.on(Events.EPOCH_COMPLETED)
def print_times(engine):
pbar.log_message(
f'Epoch {engine.state.epoch} done. Time per batch: {timer.value():.3f}[s]'
)
timer.reset()
trainer.run(train_loader, epochs)
class TestStand:
def __init__(self):
self.serial_manager = SerialManager()
try:
self.database = Database()
except KeyError:
print('Error loading database')
self.abort_test()
self.csv_logger = CSVLogger()
# create test log directories, select a serial port and begin the test
def start_test(self):
self.test_num = self.database.find_next_test_number()
self.csv_logger.test_num = self.test_num
print(f'\nStarting test {self.csv_logger.test_num}\n')
self.csv_logger.make_test_data_dir()
self.csv_logger.make_test_dir()
print('Scanning serial ports\n')
ports = self.serial_manager.get_available_serial_ports()
if not ports:
print('No serial ports were found')
self.quit_test()
else:
# let user select the correct serial port
print(('Choose a port from the options below.\n'
'Type the number of the port and press enter:'))
for port in ports:
print(ports.index(port) + 1, ' - ', port)
choice = input()
self.port = ports[int(choice) - 1]
print(f'Press enter to start test {self.test_num}\n')
input()
self.run_test()
# connect to the serial port, start the listener thread
def run_test(self):
self.serial_manager.open_port(self.port)
print("Test started!")
try:
while True:
# log serial telemetry to the influx database
data = self.serial_manager.read_telemetry()
if data:
data['test_number'] = self.test_num
self.database.log_data(data)
except KeyboardInterrupt:
self.finish_test()
# allow the user to make notes about the test, then
# log the test's notes and data inside it's directory
def finish_test(self):
print()
print("Ending test")
results = self.database.export_test(self.test_num)
if results:
self.csv_logger.log_as_csv(results)
def abort_test(self):
try:
self.csv_logger.delete_test_files()
except AttributeError:
pass
sys.exit()
def experiment():
parser = argparse.ArgumentParser(description='CNN Hyperparameter Fine-tuning')
parser.add_argument('--dataset', default='cifar10', choices=['cifar10', 'cifar100'],
help='Choose a dataset')
parser.add_argument('--model', default='resnet18', choices=['resnet18', 'wideresnet'],
help='Choose a model')
parser.add_argument('--num_finetune_epochs', type=int, default=200,
help='Number of fine-tuning epochs')
parser.add_argument('--lr', type=float, default=0.1,
help='Learning rate')
parser.add_argument('--optimizer', type=str, default='sgdm',
help='Choose an optimizer')
parser.add_argument('--batch_size', type=int, default=128,
help='Mini-batch size')
parser.add_argument('--data_augmentation', action='store_true', default=True,
help='Whether to use data augmentation')
parser.add_argument('--wdecay', type=float, default=5e-4,
help='Amount of weight decay')
parser.add_argument('--load_checkpoint', type=str,
help='Path to pre-trained checkpoint to load and finetune')
parser.add_argument('--save_dir', type=str, default='finetuned_checkpoints',
help='Save directory for the fine-tuned checkpoint')
args = parser.parse_args()
args.load_checkpoint = '/h/lorraine/PycharmProjects/CG_IFT_test/baseline_checkpoints/cifar10_resnet18_sgdm_lr0.1_wd0.0005_aug0.pt'
if args.dataset == 'cifar10':
num_classes = 10
train_loader, val_loader, test_loader = data_loaders.load_cifar10(args.batch_size, val_split=True,
augmentation=args.data_augmentation)
elif args.dataset == 'cifar100':
num_classes = 100
train_loader, val_loader, test_loader = data_loaders.load_cifar100(args.batch_size, val_split=True,
augmentation=args.data_augmentation)
if args.model == 'resnet18':
cnn = ResNet18(num_classes=num_classes)
elif args.model == 'wideresnet':
cnn = WideResNet(depth=28, num_classes=num_classes, widen_factor=10, dropRate=0.3)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
test_id = '{}_{}_{}_lr{}_wd{}_aug{}'.format(args.dataset, args.model, args.optimizer, args.lr, args.wdecay,
int(args.data_augmentation))
filename = os.path.join(args.save_dir, test_id + '.csv')
csv_logger = CSVLogger(
fieldnames=['epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'test_loss', 'test_acc'],
filename=filename)
checkpoint = torch.load(args.load_checkpoint)
init_epoch = checkpoint['epoch']
cnn.load_state_dict(checkpoint['model_state_dict'])
model = cnn.cuda()
model.train()
args.hyper_train = 'augment' # 'all_weight' # 'weight'
def init_hyper_train(model):
"""
:return:
"""
init_hyper = None
if args.hyper_train == 'weight':
init_hyper = np.sqrt(args.wdecay)
model.weight_decay = Variable(torch.FloatTensor([init_hyper]).cuda(), requires_grad=True)
model.weight_decay = model.weight_decay.cuda()
elif args.hyper_train == 'all_weight':
num_p = sum(p.numel() for p in model.parameters())
weights = np.ones(num_p) * np.sqrt(args.wdecay)
model.weight_decay = Variable(torch.FloatTensor(weights).cuda(), requires_grad=True)
model.weight_decay = model.weight_decay.cuda()
model = model.cuda()
return init_hyper
if args.hyper_train == 'augment': # Dont do inside the prior function, else scope is wrong
augment_net = UNet(in_channels=3,
n_classes=3,
depth=5,
wf=6,
padding=True,
batch_norm=False,
up_mode='upconv') # TODO(PV): Initialize UNet properly
augment_net = augment_net.cuda()
def get_hyper_train():
"""
:return:
"""
if args.hyper_train == 'weight' or args.hyper_train == 'all_weight':
return [model.weight_decay]
if args.hyper_train == 'augment':
return augment_net.parameters()
def get_hyper_train_flat():
return torch.cat([p.view(-1) for p in get_hyper_train()])
# TODO: Check this size
init_hyper_train(model)
if args.hyper_train == 'all_weight':
wdecay = 0.0
else:
wdecay = args.wdecay
optimizer = optim.SGD(model.parameters(), lr=args.lr * 0.2 * 0.2, momentum=0.9, nesterov=True,
weight_decay=wdecay) # args.wdecay)
# print(checkpoint['optimizer_state_dict'])
# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler = MultiStepLR(optimizer, milestones=[60, 120], gamma=0.2) # [60, 120, 160]
hyper_optimizer = torch.optim.Adam(get_hyper_train(), lr=1e-3) # try 0.1 as lr
# Set random regularization hyperparameters
# data_augmentation_hparams = {} # Random values for hue, saturation, brightness, contrast, rotation, etc.
if args.dataset == 'cifar10':
num_classes = 10
train_loader, val_loader, test_loader = data_loaders.load_cifar10(args.batch_size, val_split=True,
augmentation=args.data_augmentation)
elif args.dataset == 'cifar100':
num_classes = 100
train_loader, val_loader, test_loader = data_loaders.load_cifar100(args.batch_size, val_split=True,
augmentation=args.data_augmentation)
def test(loader):
model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
correct = 0.
total = 0.
losses = []
for images, labels in loader:
images = images.cuda()
labels = labels.cuda()
with torch.no_grad():
pred = model(images)
xentropy_loss = F.cross_entropy(pred, labels)
losses.append(xentropy_loss.item())
pred = torch.max(pred.data, 1)[1]
total += labels.size(0)
correct += (pred == labels).sum().item()
avg_loss = float(np.mean(losses))
acc = correct / total
model.train()
return avg_loss, acc
def prepare_data(x, y):
"""
:param x:
:param y:
:return:
"""
x, y = x.cuda(), y.cuda()
# x, y = Variable(x), Variable(y)
return x, y
def train_loss_func(x, y):
"""
:param x:
:param y:
:return:
"""
x, y = prepare_data(x, y)
reg_loss = 0.0
if args.hyper_train == 'weight':
pred = model(x)
xentropy_loss = F.cross_entropy(pred, y)
# print(f"weight_decay: {torch.exp(model.weight_decay).shape}")
for p in model.parameters():
# print(f"weight_decay: {torch.exp(model.weight_decay).shape}")
# print(f"shape: {p.shape}")
reg_loss = reg_loss + .5 * (model.weight_decay ** 2) * torch.sum(p ** 2)
# print(f"reg_loss: {reg_loss}")
elif args.hyper_train == 'all_weight':
pred = model(x)
xentropy_loss = F.cross_entropy(pred, y)
count = 0
for p in model.parameters():
reg_loss = reg_loss + .5 * torch.sum(
(model.weight_decay[count: count + p.numel()] ** 2) * torch.flatten(p ** 2))
count += p.numel()
elif args.hyper_train == 'augment':
augmented_x = augment_net(x)
pred = model(augmented_x)
xentropy_loss = F.cross_entropy(pred, y)
return xentropy_loss + reg_loss, pred
def val_loss_func(x, y):
"""
:param x:
:param y:
:return:
"""
x, y = prepare_data(x, y)
pred = model(x)
xentropy_loss = F.cross_entropy(pred, y)
return xentropy_loss
for epoch in range(init_epoch, init_epoch + args.num_finetune_epochs):
xentropy_loss_avg = 0.
total_val_loss = 0.
correct = 0.
total = 0.
progress_bar = tqdm(train_loader)
for i, (images, labels) in enumerate(progress_bar):
progress_bar.set_description('Finetune Epoch ' + str(epoch))
# TODO: Take a hyperparameter step here
optimizer.zero_grad(), hyper_optimizer.zero_grad()
val_loss, weight_norm, grad_norm = hyper_step(1, 1, get_hyper_train, get_hyper_train_flat,
model, val_loss_func,
val_loader, train_loss_func, train_loader,
hyper_optimizer)
# del val_loss
# print(f"hyper: {get_hyper_train()}")
images, labels = images.cuda(), labels.cuda()
# pred = model(images)
# xentropy_loss = F.cross_entropy(pred, labels)
xentropy_loss, pred = train_loss_func(images, labels)
optimizer.zero_grad(), hyper_optimizer.zero_grad()
xentropy_loss.backward()
optimizer.step()
xentropy_loss_avg += xentropy_loss.item()
# Calculate running average of accuracy
pred = torch.max(pred.data, 1)[1]
total += labels.size(0)
correct += (pred == labels.data).sum().item()
accuracy = correct / total
progress_bar.set_postfix(
train='%.5f' % (xentropy_loss_avg / (i + 1)),
val='%.4f' % (total_val_loss / (i + 1)),
acc='%.4f' % accuracy,
weight='%.2f' % weight_norm,
update='%.3f' % grad_norm)
val_loss, val_acc = test(val_loader)
test_loss, test_acc = test(test_loader)
tqdm.write('val loss: {:6.4f} | val acc: {:6.4f} | test loss: {:6.4f} | test_acc: {:6.4f}'.format(
val_loss, val_acc, test_loss, test_acc))
scheduler.step(epoch)
row = {'epoch': str(epoch),
'train_loss': str(xentropy_loss_avg / (i + 1)), 'train_acc': str(accuracy),
'val_loss': str(val_loss), 'val_acc': str(val_acc),
'test_loss': str(test_loss), 'test_acc': str(test_acc)}
csv_logger.writerow(row)
def main(args):
device = 'cuda:0' if args['data.cuda'] else 'cpu'
args['log.exp_dir'] = args['log.exp_dir']
if not os.path.isdir(args['log.exp_dir']):
os.makedirs(args['log.exp_dir'])
# save opts
with open(os.path.join(args['log.exp_dir'], 'args.json'), 'w') as f:
json.dump(args, f)
f.write('\n')
# Loggin
iteration_fieldnames = ['global_iteration', 'val_acc']
iteration_logger = CSVLogger(every=0,
fieldnames=iteration_fieldnames,
filename=os.path.join(args['log.exp_dir'],
'iteration_log.csv'))
# Set the random seed manually for reproducibility.
np.random.seed(args['seed'])
torch.manual_seed(args['seed'])
if args['data.cuda']:
torch.cuda.manual_seed(args['seed'])
if args['data.dataset'] == 'omniglot':
raise
train_tr = None
test_tr = None
elif args['data.dataset'] == 'miniimagenet':
train_data = get_dataset('miniimagenet-train-train', args['dataroot'])
val_data = get_dataset('miniimagenet-val', args['dataroot'])
test_data = get_dataset('miniimagenet-test', args['dataroot'])
train_tr = get_transform(
'cifar_augment_normalize_84'
if args['data_augmentation'] else 'cifar_normalize')
test_tr = get_transform('cifar_normalize')
elif args['data.dataset'] == 'cifar100':
train_data = get_dataset('cifar-fs-train-train')
val_data = get_dataset('cifar-fs-val')
test_data = get_dataset('cifar-fs-test')
train_tr = get_transform(
'cifar_augment_normalize'
if args['data_augmentation'] else 'cifar_normalize')
test_tr = get_transform('cifar_normalize')
else:
raise
model = protonet.create_model(**args)
# Load model
loaded = torch.load(args['model.model_path'])
if not 'Protonet' in str(loaded.__class__):
pretrained = ResNetClassifier(64, train_data['im_size']).to(device)
pretrained.load_state_dict(loaded)
model.encoder = pretrained.encoder
else:
model = loaded
model = BaselineFinetune(model.encoder,
args['data.way'],
args['data.shot'],
loss_type='dist')
model = model.to(device)
def evaluate(data):
corrects = []
for _ in tqdm(range(args['data.test_episodes'])):
sample = load_episode(data, test_tr, args['data.test_way'],
args['data.test_shot'],
args['data.test_query'], device)
corrects.append(classification_accuracy(sample, model)[0])
acc = torch.mean(torch.cat(corrects))
return acc.item()
print("Val acc: ", evaluate(val_data))
print("Test acc: ", evaluate(test_data))
sys.exit()
"{0}.csv".format(datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
BASE_DIR = '/sys/bus/w1/devices/'
DEVICE_FOLDER = glob.glob(BASE_DIR + '28*')[0]
DEVICE_FILE = DEVICE_FOLDER + '/w1_slave'
RELAY_PIN = 17
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
heating_relay = HeatingRelay(RELAY_PIN)
def signal_handler(signal, frame):
heating_relay.cleanup()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
thermostat = Thermostat(TempSensorReader(DEVICE_FILE))
thermostat.set_target_temperature(thermostat.read_current_temperature() +
TARGET_TEMPERATURE_CHANGE)
thermo_logger = ThermoLogger()
thermo_logger.set_csv_logger(CSVLogger(LOG_FILENAME))
thermo_logger.set_thermostat(thermostat)
thermo_logger.set_heating_relay(heating_relay)
thermo_logger.run()
def main(opt):
# Logging
trace_file = os.path.join(opt['output_dir'],
'{}_trace.txt'.format(opt['exp_name']))
# Load data
if opt['dataset'] == 'cifar-fs':
train_data = get_dataset('cifar-fs-train-train', opt['dataroot'])
val_data = get_dataset('cifar-fs-val', opt['dataroot'])
test_data = get_dataset('cifar-fs-test', opt['dataroot'])
tr = get_transform('cifar_resize_normalize')
normalize = cifar_normalize
elif opt['dataset'] == 'miniimagenet':
train_data = get_dataset('miniimagenet-train-train', opt['dataroot'])
val_data = get_dataset('miniimagenet-val', opt['dataroot'])
test_data = get_dataset('miniimagenet-test', opt['dataroot'])
tr = get_transform('cifar_resize_normalize_84')
normalize = cifar_normalize
if opt['input_regularization'] == 'oe':
reg_data = load_ood_data({
'name': 'tinyimages',
'ood_scale': 1,
'n_anom': 50000,
})
if not opt['ooe_only']:
if opt['db']:
ood_distributions = ['ooe', 'gaussian']
else:
ood_distributions = [
'ooe', 'gaussian', 'rademacher', 'texture3', 'svhn',
'tinyimagenet', 'lsun'
]
if opt['input_regularization'] == 'oe':
ood_distributions.append('tinyimages')
ood_tensors = [('ooe', None)] + [(out_name,
load_ood_data({
'name': out_name,
'ood_scale': 1,
'n_anom': 10000,
}))
for out_name in ood_distributions[1:]]
# Load trained model
loaded = torch.load(opt['model.model_path'])
if not isinstance(loaded, OrderedDict):
fs_model = loaded
else:
classifier = ResNetClassifier(64, train_data['im_size']).to(device)
classifier.load_state_dict(loaded)
fs_model = Protonet(classifier.encoder)
fs_model.eval()
fs_model = fs_model.to(device)
# Init Confidence Methods
if opt['confidence_method'] == 'oec':
init_sample = load_episode(train_data, tr, opt['data.test_way'],
opt['data.test_shot'],
opt['data.test_query'], device)
conf_model = OECConfidence(None, fs_model, init_sample, opt)
elif opt['confidence_method'] == 'deep-oec':
init_sample = load_episode(train_data, tr, opt['data.test_way'],
opt['data.test_shot'],
opt['data.test_query'], device)
conf_model = DeepOECConfidence(None, fs_model, init_sample, opt)
elif opt['confidence_method'] == 'dm-iso':
encoder = fs_model.encoder
deep_mahala_obj = DeepMahala(None,
None,
None,
encoder,
device,
num_feats=encoder.depth,
num_classes=train_data['n_classes'],
pretrained_path="",
fit=False,
normalize=None)
conf_model = DMConfidence(deep_mahala_obj, {
'ls': range(encoder.depth),
'reduction': 'max',
'g_magnitude': .1
}, True, 'iso')
if opt['pretrained_oec_path']:
conf_model.load_state_dict(torch.load(opt['pretrained_oec_path']))
conf_model.to(device)
print(conf_model)
optimizer = optim.Adam(conf_model.confidence_parameters(),
lr=opt['lr'],
weight_decay=opt['wd'])
scheduler = StepLR(optimizer,
step_size=opt['lrsche_step_size'],
gamma=opt['lrsche_gamma'])
num_param = sum(p.numel() for p in conf_model.confidence_parameters())
print(f"Learning Confidence, Number of Parameters -- {num_param}")
if conf_model.pretrain_parameters() is not None:
pretrain_optimizer = optim.Adam(conf_model.pretrain_parameters(),
lr=10)
pretrain_iter = 100
start_idx = 0
if opt['resume']:
last_ckpt_path = os.path.join(opt['output_dir'], 'last_ckpt.pt')
if os.path.exists(last_ckpt_path):
try:
last_ckpt = torch.load(last_ckpt_path)
if 'conf_model' in last_ckpt:
conf_model = last_ckpt['conf_model']
else:
sd = last_ckpt['conf_model_sd']
conf_model.load_state_dict(sd)
optimizer = last_ckpt['optimizer']
pretrain_optimizer = last_ckpt['pretrain_optimizer']
scheduler = last_ckpt['scheduler']
start_idx = last_ckpt['outer_idx']
conf_model.to(device)
except EOFError:
print(
"\n\nResuming but got EOF error, starting from init..\n\n")
wandb.run.name = opt['exp_name']
wandb.run.save()
# try:
wandb.watch(conf_model)
# except: # resuming a run
# pass
# Eval and Logging
confs = {
opt['confidence_method']: conf_model,
}
if opt['confidence_method'] == 'oec':
confs['ed'] = FSCConfidence(fs_model, 'ed')
elif opt['confidence_method'] == 'deep-oec':
encoder = fs_model.encoder
deep_mahala_obj = DeepMahala(None,
None,
None,
encoder,
device,
num_feats=encoder.depth,
num_classes=train_data['n_classes'],
pretrained_path="",
fit=False,
normalize=None)
confs['dm'] = DMConfidence(deep_mahala_obj, {
'ls': range(encoder.depth),
'reduction': 'max',
'g_magnitude': 0
}, True, 'iso').to(device)
# Temporal Ensemble for Evaluation
if opt['n_ensemble'] > 1:
nets = [deepcopy(conf_model) for _ in range(opt['n_ensemble'])]
confs['mixture-' + opt['confidence_method']] = Ensemble(
nets, 'mixture')
confs['poe-' + opt['confidence_method']] = Ensemble(nets, 'poe')
ensemble_update_interval = opt['eval_every_outer'] // opt['n_ensemble']
iteration_fieldnames = ['global_iteration']
for c in confs:
iteration_fieldnames += [
f'{c}_train_ooe', f'{c}_val_ooe', f'{c}_test_ooe', f'{c}_ood'
]
iteration_logger = CSVLogger(every=0,
fieldnames=iteration_fieldnames,
filename=os.path.join(opt['output_dir'],
'iteration_log.csv'))
best_val_ooe = 0
PATIENCE = 5 # Number of evaluations to wait
waited = 0
progress_bar = tqdm(range(start_idx, opt['train_iter']))
for outer_idx in progress_bar:
sample = load_episode(train_data, tr, opt['data.test_way'],
opt['data.test_shot'], opt['data.test_query'],
device)
conf_model.train()
if opt['full_supervision']: # sanity check
conf_model.support(sample['xs'])
in_score = conf_model.score(sample['xq'], detach=False).squeeze()
out_score = conf_model.score(sample['ooc_xq'],
detach=False).squeeze()
out_scores = [out_score]
for curr_ood, ood_tensor in ood_tensors:
if curr_ood == 'ooe':
continue
start = outer_idx % (len(ood_tensor) // 2)
stop = min(
start + sample['xq'].shape[0] * sample['xq'].shape[0],
len(ood_tensor) // 2)
oxq = torch.stack([tr(x)
for x in ood_tensor[start:stop]]).to(device)
o = conf_model.score(oxq, detach=False).squeeze()
out_scores.append(o)
#
out_score = torch.cat(out_scores)
in_score = in_score.repeat(len(ood_tensors))
loss, acc = compute_loss_bce(in_score,
out_score,
mean_center=False)
else:
conf_model.support(sample['xs'])
if opt['interpolate']:
half_n_way = sample['xq'].shape[0] // 2
interp = .5 * (sample['xq'][:half_n_way] +
sample['xq'][half_n_way:2 * half_n_way])
sample['ooc_xq'][:half_n_way] = interp
if opt['input_regularization'] == 'oe':
# Reshape ooc_xq
nw, nq, c, h, w = sample['ooc_xq'].shape
sample['ooc_xq'] = sample['ooc_xq'].view(1, nw * nq, c, h, w)
oe_bs = int(nw * nq * opt['input_regularization_percent'])
start = (outer_idx * oe_bs) % len(reg_data)
end = np.min([start + oe_bs, len(reg_data)])
oe_batch = torch.stack([tr(x) for x in reg_data[start:end]
]).to(device)
oe_batch = oe_batch.unsqueeze(0)
sample['ooc_xq'][:, :oe_batch.shape[1]] = oe_batch
if opt['in_out_1_batch']:
inps = torch.cat([sample['xq'], sample['ooc_xq']], 1)
scores = conf_model.score(inps, detach=False).squeeze()
in_score, out_score = scores[:sample['xq'].shape[1]], scores[
sample['xq'].shape[1]:]
else:
in_score = conf_model.score(sample['xq'],
detach=False).squeeze()
out_score = conf_model.score(sample['ooc_xq'],
detach=False).squeeze()
loss, acc = compute_loss_bce(in_score,
out_score,
mean_center=False)
if conf_model.pretrain_parameters(
) is not None and outer_idx < pretrain_iter:
pretrain_optimizer.zero_grad()
loss.backward()
pretrain_optimizer.step()
else:
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
progress_bar.set_postfix(loss='{:.3e}'.format(loss),
acc='{:.3e}'.format(acc))
# Update Ensemble
if opt['n_ensemble'] > 1 and outer_idx % ensemble_update_interval == 0:
update_ind = (outer_idx //
ensemble_update_interval) % opt['n_ensemble']
if opt['db']:
print(f"===> Updating Ensemble: {update_ind}")
confs['mixture-' +
opt['confidence_method']].nets[update_ind] = deepcopy(
conf_model)
confs['poe-' +
opt['confidence_method']].nets[update_ind] = deepcopy(
conf_model)
# AUROC eval
if outer_idx % opt['eval_every_outer'] == 0:
if not opt['eval_in_train']:
conf_model.eval()
# Eval..
stats_dict = {'global_iteration': outer_idx}
for conf_name, conf in confs.items():
conf.eval()
# OOE eval
ooe_aurocs = {}
for split, in_data in [('train', train_data),
('val', val_data), ('test', test_data)]:
auroc = np.mean(
eval_ood_aurocs(
None,
in_data,
tr,
opt['data.test_way'],
opt['data.test_shot'],
opt['data.test_query'],
opt['data.test_episodes'],
device,
conf,
no_grad=False
if opt['confidence_method'].startswith('dm') else
True)['aurocs'])
ooe_aurocs[split] = auroc
print_str = '{}, iter: {} ({}), auroc: {:.3e}'.format(
conf_name, outer_idx, split, ooe_aurocs[split])
_print_and_log(print_str, trace_file)
stats_dict[f'{conf_name}_train_ooe'] = ooe_aurocs['train']
stats_dict[f'{conf_name}_val_ooe'] = ooe_aurocs['val']
stats_dict[f'{conf_name}_test_ooe'] = ooe_aurocs['test']
# OOD eval
if not opt['ooe_only']:
aurocs = []
for curr_ood, ood_tensor in ood_tensors:
auroc = np.mean(
eval_ood_aurocs(
ood_tensor,
test_data,
tr,
opt['data.test_way'],
opt['data.test_shot'],
opt['data.test_query'],
opt['data.test_episodes'],
device,
conf,
no_grad=False
if opt['confidence_method'].startswith('dm')
else True)['aurocs'])
aurocs.append(auroc)
print_str = '{}, iter: {} ({}), auroc: {:.3e}'.format(
conf_name, outer_idx, curr_ood, auroc)
_print_and_log(print_str, trace_file)
mean_ood_auroc = np.mean(aurocs)
print_str = '{}, iter: {} (OOD_mean), auroc: {:.3e}'.format(
conf_name, outer_idx, mean_ood_auroc)
_print_and_log(print_str, trace_file)
stats_dict[f'{conf_name}_ood'] = mean_ood_auroc
iteration_logger.writerow(stats_dict)
plot_csv(iteration_logger.filename, iteration_logger.filename)
wandb.log(stats_dict)
if stats_dict[f'{opt["confidence_method"]}_val_ooe'] > best_val_ooe:
conf_model.cpu()
torch.save(
conf_model.state_dict(),
os.path.join(opt['output_dir'],
opt['exp_name'] + '_conf_best.pt'))
conf_model.to(device)
# Ckpt ensemble
if opt['n_ensemble'] > 1:
ensemble = confs['mixture-' + opt['confidence_method']]
ensemble.cpu()
torch.save(
ensemble.state_dict(),
os.path.join(opt['output_dir'],
opt['exp_name'] + '_ensemble_best.pt'))
ensemble.to(device)
waited = 0
else:
waited += 1
if waited >= PATIENCE:
print("PATIENCE exceeded...exiting")
sys.exit()
# For `resume`
conf_model.cpu()
torch.save(
{
'conf_model_sd':
conf_model.state_dict(),
'optimizer':
optimizer,
'pretrain_optimizer':
pretrain_optimizer
if conf_model.pretrain_parameters() is not None else None,
'scheduler':
scheduler,
'outer_idx':
outer_idx,
}, os.path.join(opt['output_dir'], 'last_ckpt.pt'))
conf_model.to(device)
conf_model.train()
sys.exit()
def __init__(
self,
# All
observation_space,
n_actions,
coords_shape,
double_replay_buffer,
task_gamma,
exploration_schedule,
seed,
learning_starts=1000,
train_freq=1,
print_freq=100,
env_name='ENV',
agent_name='AGENT',
renderer_viewer=True,
# DQN:
dqn_q_func=None,
dqn_lr=5e-4,
dqn_batch_size=32,
dqn_optim_iters=1,
dqn_target_net_update_freq=500,
dqn_grad_norm_clip=100,
dqn_double_q=True,
# Q-Map:
q_map_model=None,
q_map_random_schedule=None,
q_map_greedy_bias=0.5,
q_map_timer_bonus=0.5,
q_map_lr=5e-4,
q_map_gamma=0.9,
q_map_n_steps=1,
q_map_batch_size=32,
q_map_optim_iters=1,
q_map_target_net_update_freq=500,
q_map_min_goal_steps=10,
q_map_max_goal_steps=20,
q_map_grad_norm_clip=1000,
q_map_double_q=True):
# All
self.observation_space = observation_space
self.n_actions = n_actions
self.coords_shape = coords_shape
self.double_replay_buffer = double_replay_buffer
self.task_gamma = task_gamma
self.exploration_schedule = exploration_schedule
self.learning_starts = learning_starts
self.train_freq = train_freq
self.print_freq = print_freq
agent_name += '-train' + str(train_freq)
# DQN
if dqn_q_func is not None:
self.use_dqn = True
agent_name += '-'
agent_name += 'DQN-lr' + str(dqn_lr) + '-freq-' + str(train_freq)
self.dqn_target_net_update_freq = dqn_target_net_update_freq
self.dqn = DQN(model=dqn_q_func,
observation_space=observation_space,
n_actions=n_actions,
gamma=task_gamma,
lr=dqn_lr,
replay_buffer=double_replay_buffer,
batch_size=dqn_batch_size,
optim_iters=dqn_optim_iters,
grad_norm_clip=dqn_grad_norm_clip,
double_q=dqn_double_q)
else:
self.use_dqn = False
# Q-MAP
if q_map_model is not None:
agent_name += '-'
agent_name += 'Q-MAP-' + q_map_model.description + '-' + str(
q_map_min_goal_steps
) + '-' + str(q_map_max_goal_steps) + '-gamma' + str(
q_map_gamma) + '-lr' + str(q_map_lr) + '-bias' + str(
q_map_greedy_bias) + '-bonus' + str(q_map_timer_bonus)
self.use_q_map = True
self.q_map_timer_bonus = q_map_timer_bonus
self.using_q_map_starts = 2 * self.learning_starts
self.q_map_random_schedule = q_map_random_schedule
self.q_map_greedy_bias = q_map_greedy_bias
self.q_map_goal_proba = 1 # TODO
self.q_map_gamma = q_map_gamma
self.q_map_target_net_update_freq = q_map_target_net_update_freq
self.q_map_min_goal_steps = q_map_min_goal_steps
self.q_map_max_goal_steps = q_map_max_goal_steps
self.q_map_min_q_value = q_map_gamma**(q_map_max_goal_steps - 1)
self.q_map_max_q_value = q_map_gamma**(q_map_min_goal_steps - 1)
self.q_map_goal = None
self.q_map_goal_timer = 0
self.q_map = Q_Map(model=q_map_model,
observation_space=observation_space,
coords_shape=coords_shape,
n_actions=n_actions,
gamma=q_map_gamma,
n_steps=q_map_n_steps,
lr=q_map_lr,
replay_buffer=double_replay_buffer,
batch_size=q_map_batch_size,
optim_iters=q_map_optim_iters,
grad_norm_clip=q_map_grad_norm_clip,
double_q=q_map_double_q)
else:
self.use_q_map = False
if not self.use_dqn and not self.use_q_map:
agent_name += 'random'
else:
self.tf_saver = tf.train.Saver()
agent_name += '-memory' + str(double_replay_buffer._maxsize)
# All
home = os.path.expanduser('~')
sub_name = 'seed-{}_{}'.format(
seed,
datetime.utcnow().strftime('%F_%H-%M-%S-%f'))
self.path = '{}/results/q-map/{}/{}/{}'.format(home, env_name,
agent_name, sub_name)
# log exploration for debugging
exploration_labels = [
'steps', 'planned exploration', 'current exploration',
'random actions', 'goal actions', 'greedy actions'
]
self.exploration_logger = CSVLogger(exploration_labels,
self.path + '/exploration')
# videos etc.
self.renderer = Q_Map_Renderer(self.path, viewer=renderer_viewer)
# path to store
self.tensorflow_path = self.path + '/tensorflow'
if not os.path.exists(self.tensorflow_path):
os.makedirs(self.tensorflow_path)
U.initialize()
self.t = 0
self.episode_rewards = []
self.random_proba = self.exploration_schedule.value(0)
self.random_freq = self.exploration_schedule.value(0)
self.greedy_freq = 1.0 - self.random_freq
self.goal_freq = 0.0
if self.use_dqn:
self.dqn.update_target()
self.seed(seed)
learn_top, y_condition,logittransform,sn)
model = model.to(device)
# ipdb.set_trace()
def generate_from_noise(batch_size):
_, c2, h, w = model.prior_h.shape
c = c2 // 2
zshape = (batch_size, c, h, w)
randz = torch.randn(zshape).to(device)
randz = torch.autograd.Variable(randz, requires_grad=True)
images = model(z= randz, y_onehot=None, temperature=1, reverse=True,batch_size=batch_size)
return images
# ipdb.set_trace()
iteration_fieldnames = ['global_iteration', 'fid']
iteration_logger = CSVLogger(fieldnames=iteration_fieldnames,
filename=os.path.join(output_dir, 'eval_log.csv'))
# for idx in tqdm(np.arange(10,100,10)):
for _ in range(1):
idx = 100
model(xs[:10].cuda(), None) # this is to initialize the u,v buffer in SpectralNorm blah...otherwise state_dicts don't match...
saved_model = os.path.join(exp_dir, f"glow_model_{idx}.pth")
model.load_state_dict(torch.load(saved_model))
model.set_actnorm_init()
# Sample
with torch.no_grad():
fake = torch.cat([generate_from_noise(100) for _ in range(20)],0 )
x_is = 2*fake
x_is = x_is.repeat(1,3,1,1).detach()
def cnn_val_loss(config={}, reporter=None, callback=None, return_all=False):
print("Starting cnn_val_loss...")
###############################################################################
# Arguments
###############################################################################
dataset_options = ['cifar10', 'cifar100', 'fashion']
## Tuning parameters: all of the dropouts
parser = argparse.ArgumentParser(description='CNN')
parser.add_argument('--dataset',
default='cifar10',
choices=dataset_options,
help='Choose a dataset (cifar10, cifar100)')
parser.add_argument(
'--model',
default='resnet32',
choices=['resnet32', 'wideresnet', 'simpleconvnet'],
help='Choose a model (resnet32, wideresnet, simpleconvnet)')
#### Optimization hyperparameters
parser.add_argument('--batch_size',
type=int,
default=128,
help='Input batch size for training (default: 128)')
parser.add_argument('--epochs',
type=int,
default=int(config['epochs']),
help='Number of epochs to train (default: 200)')
parser.add_argument('--lr',
type=float,
default=float(config['lr']),
help='Learning rate')
parser.add_argument('--momentum',
type=float,
default=float(config['momentum']),
help='Nesterov momentum')
parser.add_argument('--lr_decay',
type=float,
default=float(config['lr_decay']),
help='Factor by which to multiply the learning rate.')
# parser.add_argument('--weight_decay', type=float, default=float(config['weight_decay']),
# help='Amount of weight decay to use.')
# parser.add_argument('--dropout', type=float, default=config['dropout'] if 'dropout' in config else 0.0,
# help='Amount of dropout for wideresnet')
# parser.add_argument('--dropout1', type=float, default=config['dropout1'] if 'dropout1' in config else -1,
# help='Amount of dropout for wideresnet')
# parser.add_argument('--dropout2', type=float, default=config['dropout2'] if 'dropout2' in config else -1,
# help='Amount of dropout for wideresnet')
# parser.add_argument('--dropout3', type=float, default=config['dropout3'] if 'dropout3' in config else -1,
# help='Amount of dropout for wideresnet')
parser.add_argument('--dropout_type',
type=str,
default=config['dropout_type'],
help='Type of dropout (bernoulli or gaussian)')
# Data augmentation hyperparameters
parser.add_argument(
'--inscale',
type=float,
default=0 if 'inscale' not in config else config['inscale'],
help='defines input scaling factor')
parser.add_argument('--hue',
type=float,
default=0. if 'hue' not in config else config['hue'],
help='hue jitter rate')
parser.add_argument(
'--brightness',
type=float,
default=0. if 'brightness' not in config else config['brightness'],
help='brightness jitter rate')
parser.add_argument(
'--saturation',
type=float,
default=0. if 'saturation' not in config else config['saturation'],
help='saturation jitter rate')
parser.add_argument(
'--contrast',
type=float,
default=0. if 'contrast' not in config else config['contrast'],
help='contrast jitter rate')
# Weight decay and dropout hyperparameters for each layer
parser.add_argument(
'--weight_decays',
type=str,
default='0.0',
help=
'Amount of weight decay to use for each layer, represented as a comma-separated string of floats.'
)
parser.add_argument(
'--dropouts',
type=str,
default='0.0',
help=
'Dropout rates for each layer, represented as a comma-separated string of floats'
)
parser.add_argument(
'--nonmono',
'-nonm',
type=int,
default=60,
help='how many previous epochs to consider for nonmonotonic criterion')
parser.add_argument(
'--patience',
type=int,
default=75,
help=
'How long to wait for the val loss to improve before early stopping.')
parser.add_argument(
'--data_augmentation',
action='store_true',
default=config['data_augmentation'],
help='Augment data by cropping and horizontal flipping')
parser.add_argument(
'--log_interval',
type=int,
default=10,
help='how many steps before logging stats from training set')
parser.add_argument(
'--valid_log_interval',
type=int,
default=50,
help='how many steps before logging stats from validations set')
parser.add_argument('--no_cuda',
action='store_true',
default=False,
help='enables CUDA training')
parser.add_argument('--save',
action='store_true',
default=False,
help='whether to save current run')
parser.add_argument('--seed',
type=int,
default=11,
help='random seed (default: 11)')
parser.add_argument(
'--save_dir',
default=config['save_dir'],
help=
'subdirectory of logdir/savedir to save in (default changes to date/time)'
)
args, unknown = parser.parse_known_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if args.cuda else "cpu")
cudnn.benchmark = True # Should make training should go faster for large models
torch.manual_seed(args.seed)
np.random.seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
print(args)
sys.stdout.flush()
# args.dropout1 = args.dropout1 if args.dropout1 != -1 else args.dropout
# args.dropout2 = args.dropout2 if args.dropout2 != -1 else args.dropout
# args.dropout3 = args.dropout3 if args.dropout3 != -1 else args.dropout
###############################################################################
# Saving
###############################################################################
timestamp = '{:%Y-%m-%d}'.format(datetime.datetime.now())
random_hash = random.getrandbits(16)
exp_name = '{}-dset:{}-model:{}-seed:{}-hash:{}'.format(
timestamp, args.dataset, args.model,
args.seed if args.seed else 'None', random_hash)
dropout_rates = [float(value) for value in args.dropouts.split(',')]
weight_decays = [float(value) for value in args.weight_decays.split(',')]
# Create log folder
BASE_SAVE_DIR = 'experiments'
save_dir = os.path.join(BASE_SAVE_DIR, args.save_dir, exp_name)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# Check whether the result.csv file exists already
if os.path.exists(os.path.join(save_dir, 'result.csv')):
if not args.overwrite:
print(
'The result file {} exists! Run with --overwrite to overwrite this experiment.'
.format(os.path.join(save_dir, 'result.csv')))
sys.exit(0)
# Save command-line arguments
with open(os.path.join(save_dir, 'args.yaml'), 'w') as f:
yaml.dump(vars(args), f)
epoch_csv_logger = CSVLogger(
fieldnames=['epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc'],
filename=os.path.join(save_dir, 'epoch_log.csv'))
###############################################################################
# Data Loading/Model/Optimizer
###############################################################################
if args.dataset == 'cifar10':
train_loader, valid_loader, test_loader = data_loaders.load_cifar10(
args,
args.batch_size,
val_split=True,
augmentation=args.data_augmentation)
num_classes = 10
elif args.dataset == 'cifar100':
train_loader, valid_loader, test_loader = data_loaders.load_cifar100(
args,
args.batch_size,
val_split=True,
augmentation=args.data_augmentation)
num_classes = 100
elif args.dataset == 'fashion':
train_loader, valid_loader, test_loader = data_loaders.load_fashion_mnist(
args.batch_size, val_split=True)
num_classes = 10
if args.model == 'resnet32':
cnn = resnet_cifar.resnet32(dropRates=dropout_rates)
elif args.model == 'wideresnet':
cnn = wide_resnet.WideResNet(depth=16,
num_classes=num_classes,
widen_factor=8,
dropRates=dropout_rates,
dropType=args.dropout_type)
# cnn = wide_resnet.WideResNet(depth=28, num_classes=num_classes, widen_factor=10, dropRate=args.dropout)
elif args.model == 'simpleconvnet':
cnn = models.SimpleConvNet(dropType=args.dropout_type,
conv_drop1=args.dropout1,
conv_drop2=args.dropout2,
fc_drop=args.dropout3)
def optim_parameters(model):
module_list = [
m for m in model.modules()
if type(m) == nn.Linear or type(m) == nn.Conv2d
]
weight_decays = [1e-4] * len(module_list)
return [{
'params': layer.parameters(),
'weight_decay': wdecay
} for (layer, wdecay) in zip(module_list, weight_decays)]
cnn = cnn.to(device)
criterion = nn.CrossEntropyLoss()
# cnn_optimizer = torch.optim.SGD(cnn.parameters(),
# lr=args.lr,
# momentum=args.momentum,
# nesterov=True,
# weight_decay=args.weight_decay)
cnn_optimizer = torch.optim.SGD(optim_parameters(cnn),
lr=args.lr,
momentum=args.momentum,
nesterov=True)
###############################################################################
# Training/Evaluation
###############################################################################
def evaluate(loader):
"""Returns the loss and accuracy on the entire validation/test set."""
cnn.eval()
correct = total = loss = 0.
with torch.no_grad():
for images, labels in loader:
images, labels = images.to(device), labels.to(device)
pred = cnn(images)
loss += F.cross_entropy(pred, labels, reduction='sum').item()
hard_pred = torch.max(pred, 1)[1]
total += labels.size(0)
correct += (hard_pred == labels).sum().item()
accuracy = correct / total
mean_loss = loss / total
cnn.train()
return mean_loss, accuracy
epoch = 1
global_step = 0
patience_elapsed = 0
stored_loss = 1e8
best_val_loss = []
start_time = time.time()
# This is based on the schedule used for WideResNets. The gamma (decay factor) can also be 0.2 (= 5x decay)
# Right now we're not using the scheduler because we use nonmonotonic lr decay (based on validation performance)
# scheduler = MultiStepLR(cnn_optimizer, milestones=[60,120,160], gamma=args.lr_decay)
while epoch < args.epochs + 1 and patience_elapsed < args.patience:
running_xentropy = correct = total = 0.
progress_bar = tqdm(train_loader)
for i, (images, labels) in enumerate(progress_bar):
progress_bar.set_description('Epoch ' + str(epoch))
images, labels = images.to(device), labels.to(device)
if args.inscale > 0:
noise = torch.rand(images.size(0), device=device)
scaled_noise = (
(1 + args.inscale) -
(1 / (1 + args.inscale))) * noise + (1 /
(1 + args.inscale))
images = images * scaled_noise[:, None, None, None]
# images = F.dropout(images, p=args.indropout, training=True) # TODO: Incorporate input dropout
cnn.zero_grad()
pred = cnn(images)
xentropy_loss = criterion(pred, labels)
xentropy_loss.backward()
cnn_optimizer.step()
running_xentropy += xentropy_loss.item()
# Calculate running average of accuracy
_, hard_pred = torch.max(pred, 1)
total += labels.size(0)
correct += (hard_pred == labels).sum().item()
accuracy = correct / float(total)
global_step += 1
progress_bar.set_postfix(
xentropy='%.3f' % (running_xentropy / (i + 1)),
acc='%.3f' % accuracy,
lr='%.3e' % cnn_optimizer.param_groups[0]['lr'])
val_loss, val_acc = evaluate(valid_loader)
print('Val loss: {:6.4f} | Val acc: {:6.4f}'.format(val_loss, val_acc))
sys.stdout.flush()
stats = {
'global_step': global_step,
'time': time.time() - start_time,
'loss': val_loss,
'acc': val_acc
}
# logger.write('valid', stats)
if (len(best_val_loss) > args.nonmono
and val_loss > min(best_val_loss[:-args.nonmono])):
cnn_optimizer.param_groups[0]['lr'] *= args.lr_decay
print('Decaying the learning rate to {}'.format(
cnn_optimizer.param_groups[0]['lr']))
sys.stdout.flush()
if val_loss < stored_loss:
with open(os.path.join(save_dir, 'best_checkpoint.pt'), 'wb') as f:
torch.save(cnn.state_dict(), f)
print('Saving model (new best validation)')
sys.stdout.flush()
stored_loss = val_loss
patience_elapsed = 0
else:
patience_elapsed += 1
best_val_loss.append(val_loss)
# scheduler.step(epoch)
avg_xentropy = running_xentropy / (i + 1)
train_acc = correct / float(total)
if callback is not None:
callback(epoch, avg_xentropy, train_acc, val_loss, val_acc, config)
if reporter is not None:
reporter(timesteps_total=epoch, mean_loss=val_loss)
if cnn_optimizer.param_groups[0][
'lr'] < 1e-7: # Another stopping criterion based on decaying the lr
break
epoch += 1
epoch_row = {
'epoch': str(epoch),
'train_loss': avg_xentropy,
'train_acc': str(train_acc),
'val_loss': str(val_loss),
'val_acc': str(val_acc)
}
epoch_csv_logger.writerow(epoch_row)
# Load best model and run on test
with open(os.path.join(save_dir, 'best_checkpoint.pt'), 'rb') as f:
cnn.load_state_dict(torch.load(f))
train_loss = avg_xentropy
train_acc = correct / float(total)
# Run on val and test data.
val_loss, val_acc = evaluate(valid_loader)
test_loss, test_acc = evaluate(test_loader)
print('=' * 89)
print(
'| End of training | trn loss: {:8.5f} | trn acc {:8.5f} | val loss {:8.5f} | val acc {:8.5f} | test loss {:8.5f} | test acc {:8.5f}'
.format(train_loss, train_acc, val_loss, val_acc, test_loss, test_acc))
print('=' * 89)
sys.stdout.flush()
# Save the final val and test performance to a results CSV file
with open(os.path.join(save_dir, 'result_{}.csv'.format(time.time())),
'w') as result_file:
result_writer = csv.DictWriter(result_file,
fieldnames=[
'train_loss', 'train_acc',
'val_loss', 'val_acc', 'test_loss',
'test_acc'
])
result_writer.writeheader()
result_writer.writerow({
'train_loss': train_loss,
'train_acc': train_acc,
'val_loss': val_loss,
'val_acc': val_acc,
'test_loss': test_loss,
'test_acc': test_acc
})
result_file.flush()
if return_all:
print("RETURNING ", train_loss, train_acc, val_loss, val_acc,
test_loss, test_acc)
sys.stdout.flush()
return train_loss, train_acc, val_loss, val_acc, test_loss, test_acc
else:
print("RETURNING ", stored_loss)
sys.stdout.flush()
return stored_loss
def main(args):
device = 'cuda:0' if args['data.cuda'] else 'cpu'
args['log.exp_dir'] = args['log.exp_dir']
if not os.path.isdir(args['log.exp_dir']):
os.makedirs(args['log.exp_dir'])
# save opts
with open(os.path.join(args['log.exp_dir'], 'args.json'), 'w') as f:
json.dump(args, f)
f.write('\n')
# Loggin
iteration_fieldnames = ['global_iteration', 'val_acc']
iteration_logger = CSVLogger(every=0,
fieldnames=iteration_fieldnames,
filename=os.path.join(args['log.exp_dir'],
'iteration_log.csv'))
# Set the random seed manually for reproducibility.
np.random.seed(args['seed'])
torch.manual_seed(args['seed'])
if args['data.cuda']:
torch.cuda.manual_seed(args['seed'])
if args['data.dataset'] == 'omniglot':
raise
train_tr = None
test_tr = None
elif args['data.dataset'] == 'miniimagenet':
train_data = get_dataset('miniimagenet-train-train', args['dataroot'])
val_data = get_dataset('miniimagenet-val', args['dataroot'])
train_tr = get_transform(
'cifar_augment_normalize_84'
if args['data_augmentation'] else 'cifar_normalize')
test_tr = get_transform('cifar_normalize')
elif args['data.dataset'] == 'cifar100':
train_data = get_dataset('cifar-fs-train-train')
val_data = get_dataset('cifar-fs-val')
train_tr = get_transform(
'cifar_augment_normalize'
if args['data_augmentation'] else 'cifar_normalize')
test_tr = get_transform('cifar_normalize')
else:
raise
model = protonet.create_model(**args)
if args['model.model_path'] != '':
loaded = torch.load(args['model.model_path'])
if not 'Protonet' in str(loaded.__class__):
pretrained = ResNetClassifier(64, train_data['im_size']).to(device)
pretrained.load_state_dict(loaded)
model.encoder = pretrained.encoder
else:
model = loaded
model = model.to(device)
max_epoch = args['train.epochs']
epoch = 0
stop = False
patience_elapsed = 0
best_metric_value = 0.0
def evaluate():
nonlocal best_metric_value
nonlocal patience_elapsed
nonlocal stop
nonlocal epoch
corrects = []
for _ in tqdm(range(args['data.test_episodes']),
desc="Epoch {:d} Val".format(epoch + 1)):
sample = load_episode(val_data, test_tr, args['data.test_way'],
args['data.test_shot'],
args['data.test_query'], device)
corrects.append(classification_accuracy(sample, model)[0])
val_acc = torch.mean(torch.cat(corrects))
iteration_logger.writerow({
'global_iteration': epoch,
'val_acc': val_acc.item()
})
plot_csv(iteration_logger.filename, iteration_logger.filename)
print(f"Epoch {epoch}: Val Acc: {val_acc}")
if val_acc > best_metric_value:
best_metric_value = val_acc
print("==> best model (metric = {:0.6f}), saving model...".format(
best_metric_value))
model.cpu()
torch.save(model, os.path.join(args['log.exp_dir'],
'best_model.pt'))
model.to(device)
patience_elapsed = 0
else:
patience_elapsed += 1
if patience_elapsed > args['train.patience']:
print("==> patience {:d} exceeded".format(
args['train.patience']))
stop = True
optim_method = getattr(optim, args['train.optim_method'])
params = model.parameters()
optimizer = optim_method(params,
lr=args['train.learning_rate'],
weight_decay=args['train.weight_decay'])
scheduler = lr_scheduler.StepLR(optimizer,
args['train.decay_every'],
gamma=0.5)
while epoch < max_epoch and not stop:
evaluate()
model.train()
if epoch % args['ckpt_every'] == 0:
model.cpu()
torch.save(model,
os.path.join(args['log.exp_dir'], f'model_{epoch}.pt'))
model.to(device)
scheduler.step()
for _ in tqdm(range(args['data.train_episodes']),
desc="Epoch {:d} train".format(epoch + 1)):
sample = load_episode(train_data, train_tr, args['data.way'],
args['data.shot'], args['data.query'],
device)
optimizer.zero_grad()
loss, output = model.loss(sample)
loss.backward()
optimizer.step()
epoch += 1
class TestStand:
def __init__(self):
self.serial_manager = SerialManager()
try:
self.database = Database()
except KeyError:
print(
'Error loading database. Did you set the TEST_STAND_DB '
'environment variable to the name of your InfluxDB database?')
self.abort_test()
self.csv_logger = CSVLogger()
# thread for interpreting incoming serial data
self.telemetry_thread = threading.Thread(target=self.telemetry_loop)
self.thread_is_running = False
# create test log directories, select a serial port and begin the test
def start_test(self):
self.test_num = self.database.find_next_test_number()
self.csv_logger.test_num = self.test_num
print(f'\nStarting test {self.csv_logger.test_num}\n')
self.csv_logger.make_test_data_dir()
self.csv_logger.make_test_dir()
print('Scanning serial ports\n')
ports = self.serial_manager.get_available_serial_ports()
if not ports:
print('No serial ports were found')
self.quit_test()
else:
# let user select the correct serial port
print(('Choose a port from the options below.\n'
'Type the number of the port and press enter:'))
for port in ports:
print(ports.index(port) + 1, ' - ', port)
choice = input()
self.port = ports[int(choice) - 1]
print(f'Ready to start test {self.test_num}\n')
keyword = input('To begin type "start": ')
while keyword != 'start':
keyword = input('Wrong keyword. To begin type "start": ')
self.run_test()
# connect to the serial port, start the listener thread and allow the user
# to control the solenoid with the enter/return key
def run_test(self):
self.serial_manager.open_port(self.port)
self.thread_is_running = True
self.telemetry_thread.start()
print('Press enter to toggle solenoid')
while True:
if input() == 'q':
print('\n')
self.finish_test()
break
self.serial_manager.toggle_solenoid()
# drive the solenoid low, allow the user to make notes about the test, then
# log the test's notes and data inside it's directory
def finish_test(self):
if self.serial_manager.solenoid_state:
self.serial_manager.toggle_solenoid()
while self.serial_manager.solenoid_event_queued:
pass
self.thread_is_running = False
notes = []
note = input('Make any notes about this test below:\n')
while note != '':
notes.append(note)
note = input()
self.csv_logger.log_notes(notes)
results = self.database.export_test(self.test_num)
if results:
self.csv_logger.log_as_csv(results)
def abort_test(self):
self.thread_is_running = False
try:
self.csv_logger.delete_test_files()
except AttributeError:
pass
sys.exit()
def telemetry_loop(self):
while self.thread_is_running:
# if there is a toggle queued for the solenoid
if self.serial_manager.solenoid_event_queued:
# write a 0 or 1 to the serial port to toggle the solenoid
self.serial_manager.serial.write(
str(self.serial_manager.solenoid_state).encode())
self.serial_manager.solenoid_event_queued = False
# log serial telemetry to the influx database
data = self.serial_manager.read_telemetry()
if data:
data['test_number'] = self.test_num
self.database.log_data(data)
U.initialize()
plt.ion()
test_obs = []
test_qmaps = []
image_indexes = []
n_images = 20
for level in test_levels:
test_obs.append(np.load('gridworld_obs_{}.npy'.format(level)))
test_qmaps.append(np.load('gridworld_gound_truth_{}.npy'.format(level)))
image_indexes.append(np.linspace(300, len(test_obs[-1])-300, n_images).astype(int))
if args.render:
viewer = rendering.SimpleImageViewer(maxwidth=2500)
agent_name = 'Qmap_{}_gamma{}_{}{}{}lr{}_batch{}_target{}'.format(args.model, args.gamma, 'dueling_' if args.dueling else '', 'double_' if args.double else '', 'layernorm_' if args.norm else '', args.lr, args.batch, args.target)
path = '{}/{}/{}'.format(args.path, agent_name, args.seed)
loss_logger = CSVLogger(['steps'] + test_levels, os.path.expanduser('{}/ground_truth_loss'.format(path)))
os.mkdir('{}/images'.format(path))
color_map = plt.get_cmap('inferno')
batch_weights = np.ones(q_map.batch_size)
batch_dones = np.zeros((q_map.batch_size, 1))
for t in range(n_steps // q_map.batch_size + 1):
batch_prev_frames = []
batch_ac = []
batch_rcw = []
batch_frames = []
for _ in range(q_map.batch_size):
prev_ob = env.random_reset()
ac = env.action_space.sample()
ob = env.step(ac)[0]
class Q_Map_DQN_Agent(Agent):
def __init__(
self,
# All
observation_space,
n_actions,
coords_shape,
double_replay_buffer,
task_gamma,
exploration_schedule,
seed,
learning_starts=1000,
train_freq=1,
print_freq=100,
env_name='ENV',
agent_name='AGENT',
renderer_viewer=True,
# DQN:
dqn_q_func=None,
dqn_lr=5e-4,
dqn_batch_size=32,
dqn_optim_iters=1,
dqn_target_net_update_freq=500,
dqn_grad_norm_clip=100,
dqn_double_q=True,
# Q-Map:
q_map_model=None,
q_map_random_schedule=None,
q_map_greedy_bias=0.5,
q_map_timer_bonus=0.5,
q_map_lr=5e-4,
q_map_gamma=0.9,
q_map_n_steps=1,
q_map_batch_size=32,
q_map_optim_iters=1,
q_map_target_net_update_freq=500,
q_map_min_goal_steps=10,
q_map_max_goal_steps=20,
q_map_grad_norm_clip=1000,
q_map_double_q=True):
# All
self.observation_space = observation_space
self.n_actions = n_actions
self.coords_shape = coords_shape
self.double_replay_buffer = double_replay_buffer
self.task_gamma = task_gamma
self.exploration_schedule = exploration_schedule
self.learning_starts = learning_starts
self.train_freq = train_freq
self.print_freq = print_freq
agent_name += '-train' + str(train_freq)
# DQN
if dqn_q_func is not None:
self.use_dqn = True
agent_name += '-'
agent_name += 'DQN-lr' + str(dqn_lr) + '-freq-' + str(train_freq)
self.dqn_target_net_update_freq = dqn_target_net_update_freq
self.dqn = DQN(model=dqn_q_func,
observation_space=observation_space,
n_actions=n_actions,
gamma=task_gamma,
lr=dqn_lr,
replay_buffer=double_replay_buffer,
batch_size=dqn_batch_size,
optim_iters=dqn_optim_iters,
grad_norm_clip=dqn_grad_norm_clip,
double_q=dqn_double_q)
else:
self.use_dqn = False
# Q-MAP
if q_map_model is not None:
agent_name += '-'
agent_name += 'Q-MAP-' + q_map_model.description + '-' + str(
q_map_min_goal_steps
) + '-' + str(q_map_max_goal_steps) + '-gamma' + str(
q_map_gamma) + '-lr' + str(q_map_lr) + '-bias' + str(
q_map_greedy_bias) + '-bonus' + str(q_map_timer_bonus)
self.use_q_map = True
self.q_map_timer_bonus = q_map_timer_bonus
self.using_q_map_starts = 2 * self.learning_starts
self.q_map_random_schedule = q_map_random_schedule
self.q_map_greedy_bias = q_map_greedy_bias
self.q_map_goal_proba = 1 # TODO
self.q_map_gamma = q_map_gamma
self.q_map_target_net_update_freq = q_map_target_net_update_freq
self.q_map_min_goal_steps = q_map_min_goal_steps
self.q_map_max_goal_steps = q_map_max_goal_steps
self.q_map_min_q_value = q_map_gamma**(q_map_max_goal_steps - 1)
self.q_map_max_q_value = q_map_gamma**(q_map_min_goal_steps - 1)
self.q_map_goal = None
self.q_map_goal_timer = 0
self.q_map = Q_Map(model=q_map_model,
observation_space=observation_space,
coords_shape=coords_shape,
n_actions=n_actions,
gamma=q_map_gamma,
n_steps=q_map_n_steps,
lr=q_map_lr,
replay_buffer=double_replay_buffer,
batch_size=q_map_batch_size,
optim_iters=q_map_optim_iters,
grad_norm_clip=q_map_grad_norm_clip,
double_q=q_map_double_q)
else:
self.use_q_map = False
if not self.use_dqn and not self.use_q_map:
agent_name += 'random'
else:
self.tf_saver = tf.train.Saver()
agent_name += '-memory' + str(double_replay_buffer._maxsize)
# All
home = os.path.expanduser('~')
sub_name = 'seed-{}_{}'.format(
seed,
datetime.utcnow().strftime('%F_%H-%M-%S-%f'))
self.path = '{}/results/q-map/{}/{}/{}'.format(home, env_name,
agent_name, sub_name)
# log exploration for debugging
exploration_labels = [
'steps', 'planned exploration', 'current exploration',
'random actions', 'goal actions', 'greedy actions'
]
self.exploration_logger = CSVLogger(exploration_labels,
self.path + '/exploration')
# videos etc.
self.renderer = Q_Map_Renderer(self.path, viewer=renderer_viewer)
# path to store
self.tensorflow_path = self.path + '/tensorflow'
if not os.path.exists(self.tensorflow_path):
os.makedirs(self.tensorflow_path)
U.initialize()
self.t = 0
self.episode_rewards = []
self.random_proba = self.exploration_schedule.value(0)
self.random_freq = self.exploration_schedule.value(0)
self.greedy_freq = 1.0 - self.random_freq
self.goal_freq = 0.0
if self.use_dqn:
self.dqn.update_target()
self.seed(seed)
def seed(self, seed):
self.np_random, seed = seeding.np_random(seed)
if self.use_dqn:
self.dqn.seed(seed)
if self.use_q_map:
self.q_map.seed(seed)
return [seed]
def reset(self, ob):
if self.use_q_map:
self.q_map_goal_timer = 0
self.q_map_goal = None
frames = ob[0]
ac = self.choose_action(ob)
self.log()
self.episode_rewards.append(0.0)
self.prev_ob = ob
self.prev_ac = ac
return ac
def step(self, ob, rew, done):
prev_frames, (_, _, prev_w), _, _ = self.prev_ob
frames, (row, col, w), _, _ = ob
if self.double_replay_buffer is not None:
self.double_replay_buffer.add(prev_frames, self.prev_ac, rew,
(row, col - w, w - prev_w), frames,
done)
self.optimize()
if not done:
ac = self.choose_action(ob)
else:
ac = None
self.add_to_renderer(ob)
self.t += 1
self.episode_rewards[-1] += rew
self.prev_ob = ob
self.prev_ac = ac
return ac
def choose_action(self, ob):
frames, (row, col, w), screen, (full_r, full_c) = ob
q_map_values = None
q_map_candidates = []
q_map_biased_candidates = []
# render Q-maps all the time even if we do not need them
if self.use_q_map:
q_map_values = self.q_map.compute_q_values(
frames[None])[0] # (rows, cols, acs)
if self.np_random.rand() < self.random_proba or (
not self.use_dqn and self.t <= self.using_q_map_starts):
ac = self.np_random.randint(self.n_actions)
action_type = 'random'
else:
# Q-Map available and started to train
if self.use_q_map and self.t > self.using_q_map_starts:
# reached goal
if self.q_map_goal_timer > 0 and self.q_map_goal[1] < w:
self.q_map_goal_timer = 0
self.q_map_goal = None
# goal unreachable
if self.q_map_goal_timer > 0 and (row, col) == self.q_map_goal:
self.q_map_goal_timer = 0
self.q_map_goal = None
# no more goal
if self.q_map_goal_timer == 0:
if self.np_random.rand() < self.q_map_goal_proba:
# find a new goal
q_map_max_values = q_map_values.max(2) # (rows, cols)
q_map_candidates_mask = np.logical_and(
self.q_map_min_q_value <= q_map_max_values,
self.q_map_max_q_value >= q_map_max_values)
q_map_candidates = np.where(q_map_candidates_mask)
q_map_candidates = np.dstack(q_map_candidates)[
0] # list of (row, col)
if len(q_map_candidates) > 0:
# goals compatible with greedy action
if self.use_dqn and self.np_random.rand(
) < self.q_map_greedy_bias:
greedy_ac = self.dqn.choose_action(
frames, stochastic=False)
q_map_biased_candidates_mask = np.logical_and(
q_map_candidates_mask,
q_map_values.argmax(2) == greedy_ac)
q_map_biased_candidates = np.where(
q_map_biased_candidates_mask)
q_map_biased_candidates = np.dstack(
q_map_biased_candidates)[
0] # list of (row, col)
# same DQN and Q-Map action
if len(q_map_biased_candidates) > 0:
goal_idx = self.np_random.randint(
len(q_map_biased_candidates))
q_map_goal_row, q_map_goal_col_local = q_map_biased_candidates[
goal_idx]
q_map_expected_steps = math.log(
q_map_max_values[q_map_goal_row,
q_map_goal_col_local],
self.q_map_gamma) + 1
self.q_map_goal_timer = math.ceil(
1.5 * q_map_expected_steps) # 50% bonus
self.q_map_goal = (q_map_goal_row,
q_map_goal_col_local + w)
ac = greedy_ac
action_type = 'dqn/qmap'
# greedy Q-Map action
else:
goal_idx = self.np_random.randint(
len(q_map_candidates))
q_map_goal_row, q_map_goal_col_local = q_map_candidates[
goal_idx]
q_map_expected_steps = math.log(
q_map_max_values[q_map_goal_row,
q_map_goal_col_local],
self.q_map_gamma) + 1
self.q_map_goal_timer = math.ceil(
(1. + self.q_map_timer_bonus) *
q_map_expected_steps)
self.q_map_goal = (q_map_goal_row,
q_map_goal_col_local + w)
ac, q_map_values = self.q_map.choose_action(
None,
(q_map_goal_row, q_map_goal_col_local),
q_map_values
) # no need to recompute the Q-Map
action_type = 'qmap'
self.q_map_goal_timer -= 1
if self.q_map_goal_timer == 0:
self.q_map_goal = None
# random action
else:
self.q_map_goal_timer = 0
self.q_map_goal = None
ac = self.np_random.randint(self.n_actions)
action_type = 'random'
# DQN action
else:
ac = self.dqn.choose_action(frames, stochastic=False)
action_type = 'dqn'
# Q-Map action
else:
q_map_goal_row, q_map_goal_col = self.q_map_goal
q_map_goal_col_local = q_map_goal_col - w
ac, q_map_values = self.q_map.choose_action(
frames, (q_map_goal_row, q_map_goal_col_local))
self.q_map_goal_timer -= 1
if self.q_map_goal_timer == 0:
self.q_map_goal = None
action_type = 'qmap'
# DQN action
else:
ac = self.dqn.choose_action(frames, stochastic=False)
action_type = 'dqn'
# rendering
self.add_to_renderer(ob, q_map_values, ac, action_type,
q_map_candidates, q_map_biased_candidates)
# update exploration
if action_type == 'dqn/qmap':
self.random_freq += 0.01 * (0 - self.random_freq)
self.greedy_freq += 0.01 * (1 - self.greedy_freq)
self.goal_freq += 0.01 * (0 - self.goal_freq) # TODO: 1?
elif action_type == 'dqn':
self.random_freq += 0.01 * (0 - self.random_freq)
self.greedy_freq += 0.01 * (1 - self.greedy_freq)
self.goal_freq += 0.01 * (0 - self.goal_freq)
elif action_type == 'qmap':
self.random_freq += 0.01 * (0 - self.random_freq)
self.greedy_freq += 0.01 * (0 - self.greedy_freq)
self.goal_freq += 0.01 * (1 - self.goal_freq)
elif action_type == 'random':
self.random_freq += 0.01 * (1 - self.random_freq)
self.greedy_freq += 0.01 * (0 - self.greedy_freq)
self.goal_freq += 0.01 * (0 - self.goal_freq)
else:
raise NotImplementedError(
'unknown action type {}'.format(action_type))
target_exploration = self.exploration_schedule.value(self.t)
current_exploration = (1.0 - self.greedy_freq)
if self.use_q_map and self.t >= self.using_q_map_starts:
self.random_proba = self.q_map_random_schedule.value(self.t)
if current_exploration > target_exploration:
self.q_map_goal_proba -= 0.001
elif current_exploration < target_exploration:
self.q_map_goal_proba += 0.001
else:
self.random_proba = self.exploration_schedule.value(self.t)
if (self.t + 1) % 100 == 0:
self.exploration_logger.log(self.t + 1, target_exploration,
current_exploration, self.random_freq,
self.goal_freq, self.greedy_freq)
return ac
def optimize(self):
if (
self.use_dqn or self.use_q_map
) and self.t >= self.learning_starts and self.t % self.train_freq == 0:
if self.use_dqn:
self.dqn.optimize(self.t)
if self.use_q_map:
self.q_map.optimize(self.t)
if self.use_dqn and self.t >= self.learning_starts and self.t % self.dqn_target_net_update_freq == 0:
self.dqn.update_target()
if self.use_q_map and self.t >= self.learning_starts and self.t % self.q_map_target_net_update_freq == 0:
self.q_map.update_target()
# save the session
if (self.use_dqn or self.use_q_map) and (self.t + 1) % 100000 == 0:
file_name = self.tensorflow_path + '/step_' + str(self.t +
1) + '.ckpt'
print('saving tensorflow session to', file_name)
self.tf_saver.save(tf.get_default_session(), file_name)
def log(self):
if self.t > 0 and self.print_freq is not None and len(
self.episode_rewards) % self.print_freq == 0:
mean_100ep_reward = np.mean(self.episode_rewards[-100:])
num_episodes = len(self.episode_rewards)
logger.record_tabular('steps', self.t)
logger.record_tabular('episodes', num_episodes)
logger.record_tabular('mean 100 episode reward',
'{:.3f}'.format(mean_100ep_reward))
logger.record_tabular(
'exploration (target)', '{:.3f} %'.format(
100 * self.exploration_schedule.value(self.t)))
logger.record_tabular(
'exploration (current)',
'{:.3f} %'.format(100 * (1.0 - self.greedy_freq)))
logger.dump_tabular()
def load(self, path):
self.tf_saver.restore(tf.get_default_session(), path)
print('model restored :)')
def add_to_renderer(self,
ob,
q_map_values=None,
ac=None,
action_type='',
q_map_candidates=[],
q_map_biased_candidates=[]):
if self.renderer is not None:
if self.use_q_map and self.q_map_goal is not None:
goal = self.q_map_goal
assert self.q_map_goal_timer > 0
else:
goal = None
self.renderer.add(ob, self.coords_shape, q_map_values, ac,
action_type, self.n_actions, q_map_candidates,
q_map_biased_candidates, goal)
