def __init__(self, name='deathbot', load_weights=False, training=False, batch_size=100, lr=1e-3, location=None):
self.session = tf.Session()
self.name = name
if training:
from hyperdash import Experiment
self.exp = Experiment(name)
if name in MODELS.keys(): self.model = MODELS[name]() if not training else MODELS[name](self.exp)
adam = Adam(lr=lr)
nadam = Nadam(lr=lr)
#rms = RMSprop(lr=lr)
#sgd = SGD(lr=lr)
self.optimizer = adam if name == "evo" else nadam
loss = ["binary_crossentropy", "categorical_crossentropy", "poisson"]
self.model.compile(optimizer=self.optimizer, loss=loss[1],
metrics=["acc"])
self.callbacks = []
if training:
self.exp.param("lr", lr)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5,
patience=4, min_lr=1e-4, verbose=1)
tb = TensorBoard('./models/logs/{}'.format(name), write_graph=True)
cp = ModelCheckpoint(filepath='./models/weights-{}.hdf5'.format(name), monitor='val_acc', verbose=1,
save_best_only=True)
hd = Hyperdash(self.exp, self.model)
es = EarlyStopping('val_acc', patience=5, verbose=1)
self.callbacks = [cp, tb, hd, reduce_lr, es]
if load_weights:
#print(os.listdir(os.getcwd()))
self.model.load_weights('./final/weights-{}.hdf5'.format(name))
if training: print('Weights Loaded...')
def _new_exp(name, supress_output):
if os.path.exists(API_KEY_JSON_PATH):
exp = Experiment(name,
capture_io=False,
api_key_getter=_api_key_getter)
else:
exp = Experiment(name, capture_io=False)
# SUPER-hacky, but it's work (needed to supress hd output)
if supress_output:
exp._hd.out_buf.write = lambda _: _
return exp
def test_experiment_keras_callback(self):
with patch("sys.stdout", new=StringIO()) as faked_out:
exp = Experiment("MNIST")
keras_cb = exp.callbacks.keras
keras_cb.on_epoch_end(0, {"val_acc": 1, "val_loss": 2})
# Sleep 1 second due to client sampling
time.sleep(1)
keras_cb.on_epoch_end(1, {"val_acc": 3, "val_loss": 4})
exp.end()
# Test metrics match what is expected
metrics_messages = []
for msg in server_sdk_messages:
payload = msg["payload"]
if "name" in payload:
metrics_messages.append(payload)
expect_metrics = [
{
"is_internal": False,
"name": "val_acc",
"value": 1
},
{
"is_internal": False,
"name": "val_loss",
"value": 2
},
{
"is_internal": False,
"name": "val_acc",
"value": 3
},
{
"is_internal": False,
"name": "val_loss",
"value": 4
},
]
assert len(expect_metrics) == len(metrics_messages)
for i, message in enumerate(metrics_messages):
assert message["is_internal"] == expect_metrics[i]["is_internal"]
assert message["name"] == expect_metrics[i]["name"]
assert message["value"] == expect_metrics[i]["value"]
captured_out = faked_out.getvalue()
assert "error" not in captured_out
def __init__(self, dataset_name, model_name, optimizer_name, trial_num):
"""
:param dataset_name: name of the dataset
:type dataset_name: str
:param model_name: name of the model
:type model_name: str
:param optimizer_name: name of the optimizer
:type optimizer_name: str
:param trial_num: current number of repeated trials
:type trial_num: int
"""
# get optimized hyperparameters
with open(
f'../params/{dataset_name}_{model_name}_{optimizer_name}/result.json'
) as f:
params = json.load(f)
# get instances
self.dataset = Datasets.get(dataset_name)
self.model = Models.get(model_name, dataset=self.dataset)
self.optimizer = Optimizers.get(optimizer_name, params=params)
# get config
with open('./config.json') as f:
config = json.load(f)
# get constants
c = config['constants'][dataset_name][model_name]
self.loss = c['loss']
self.batch_size = c['batch_size']
self.epochs = c['epochs']
# configure and initialize directory
d = self.main_dir = f'../data/{dataset_name}_{model_name}_{optimizer_name}/trial{trial_num}'
if os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d)
# configure hyperdash experiment
self.hd_exp = HyperdashExperiment(
f'{dataset_name}',
api_key_getter=lambda: config['hyperdash']['api_key'])
self.hd_exp.param('dataset_name', dataset_name)
self.hd_exp.param('model_name', model_name)
self.hd_exp.param('optimizer_name', optimizer_name)
self.hd_exp.param('trial_num', trial_num)
for k, v in params.items():
self.hd_exp.param(k, v)
# set callbacks
self.callbacks = [
Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'],
self.hd_exp),
TensorBoard(log_dir=f'{self.main_dir}/tensorboard'),
TimeLogger(filename=f'{self.main_dir}/time.csv'),
CSVLogger(filename=f'{self.main_dir}/result.csv', append=True)
]
def main():
"""Start training."""
exp = Experiment("diffrend test")
# Parse args
opt = Parameters().parse()
for key, val in opt.__dict__.items():
exp.param(key, val)
# Create dataset loader
dataset_load = Dataset_load(opt)
# Create GAN
gan = GAN(opt, dataset_load, exp)
# Train gan
gan.train()
def prepare_experiment(exp_name, model_dir='models'):
exp = Experiment(exp_name)
model_dir = os.path.join(model_dir, exp_name, get_jstdate_string())
checkpoint_filename = os.path.join(model_dir, 'checkpoint_{epoch:02d}-{val_loss:.2f}.hdf5')
make_dirs(model_dir)
hd_callback = Hyperdash(['val_acc', 'val_loss'], exp)
checkpoint = keras.callbacks.ModelCheckpoint(
checkpoint_filename, monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=False, mode='auto', period=1)
return exp, hd_callback, checkpoint
def __init__(self, output_dir, data_loader, n_words, ixtoword):
if cfg.TRAIN.FLAG:
self.model_dir = os.path.join(output_dir, 'Model')
self.image_dir = os.path.join(output_dir, 'Image')
mkdir_p(self.model_dir)
mkdir_p(self.image_dir)
# torch.cuda.set_device(cfg.GPU_ID)
cudnn.benchmark = True
self.batch_size = cfg.TRAIN.BATCH_SIZE
self.max_epoch = cfg.TRAIN.MAX_EPOCH
self.snapshot_interval = cfg.TRAIN.SNAPSHOT_INTERVAL
self.n_words = n_words
self.ixtoword = ixtoword
self.data_loader = data_loader
self.num_batches = len(self.data_loader)
self.start_epoch = 0
self.exp = Experiment("t2s", capture_io=False, api_key_getter=get_api_key_from_env)
def pretrain_ddec(args):
print("Pretraining...")
print("Loading dataset...")
with open(os.path.join(args.text_embedding_dir, 'word_embedding.p'), "rb") as f:
embedding_model = cPickle.load(f)
with open(os.path.join(args.text_embedding_dir, 'word_idx.json'), "r", encoding='utf-8') as f:
word_idx = json.load(f)
train_dataset, test_dataset = load_pretrain_data(args.image_dir, word_idx[1], args, CONFIG)
print("Loading dataset completed")
dualnet = DualNet(pretrained_embedding=embedding_model, text_features=args.text_features, z_dim=args.z_dim, n_classes=args.n_classes)
if args.resume:
print("loading model...")
dualnet.load_model("/4TBSSD/CHECKPOINT/pretrain_" + str(args.z_dim) + "_0.pt")
exp = Experiment("Dualnet_pretrain_" + str(args.z_dim), capture_io=True)
print(dualnet)
for arg, value in vars(args).items():
exp.param(arg, value)
try:
dualnet.fit(train_dataset, test_dataset, args=args,
save_path="/4TBSSD/CHECKPOINT/pretrain_" + str(args.z_dim) + "_0.pt")
print("Finish!!!")
finally:
exp.end()
def train_multidec(args):
print("Training multidec")
device = torch.device(args.gpu)
print("Loading dataset...")
full_dataset = load_multi_csv_data(args, CONFIG)
print("Loading dataset completed")
# full_loader = DataLoader(full_dataset, batch_size=args.batch_size, shuffle=False)
image_encoder = MDEC_encoder(input_dim=args.input_dim, z_dim=args.latent_dim, n_clusters=args.n_clusters,
encodeLayer=[500, 500, 2000], activation="relu", dropout=0)
image_encoder.load_model(os.path.join(CONFIG.CHECKPOINT_PATH, "image_sdae_" + str(args.latent_dim)) + ".pt")
text_encoder = MDEC_encoder(input_dim=args.input_dim, z_dim=args.latent_dim, n_clusters=args.n_clusters,
encodeLayer=[500, 500, 2000], activation="relu", dropout=0)
text_encoder.load_model(os.path.join(CONFIG.CHECKPOINT_PATH, "text_sdae_" + str(args.latent_dim)) + ".pt")
mdec = MultiDEC(device=device, image_encoder=image_encoder, text_encoder=text_encoder, n_clusters=args.n_clusters)
exp = Experiment("MDEC " + str(args.latent_dim) + '_' + str(args.n_clusters), capture_io=True)
print(mdec)
for arg, value in vars(args).items():
exp.param(arg, value)
try:
mdec.fit(full_dataset, lr=args.lr, batch_size=args.batch_size, num_epochs=args.epochs,
save_path=CONFIG.CHECKPOINT_PATH)
print("Finish!!!")
finally:
exp.end()
class HyperdashCallback(Callback):
exp = None
last = 1
def on_train_begin(self, logs=None):
self.exp = Experiment("Deep Weather")
def on_train_end(self, logs=None):
self.exp.end()
def on_epoch_end(self, epoch, logs=None):
if 'loss' in logs:
self.exp.metric("progress", min(0.1, self.last - logs["loss"]))
self.last = logs["loss"]
self.exp.metric("loss", min(0.5, logs["loss"]))
self.exp.metric("val_loss", min(0.5, logs["val_loss"]))
def train_bayes(params):
"""
Wrapper around train function to serve as objective function for Gaussian
optimization in scikit-optimize routine gp_minimize.
Arguments:
----------
params: list, shape=[nb_layers + 2,]
List of search space dimensions. Entries have to be tuples
(lower_bound, upper_bound) for Reals or Integers.
Returns:
--------
tbd
"""
# Create Hyperdash hd_experiment
hd_exp = Experiment(project_name)
# Translate params into format understood by train function
# n_layer = 4
# layer_sizes = hd_exp.param('layer_sizes', (2**np.array(params[:n_layer])).tolist())
# learning_rate = hd_exp.param('learning rate', 10**params[n_layer])
# mini_batch_size = hd_exp.param('mini batch size', int(2**params[n_layer + 1]))
# pkeep = hd_exp.param('dropout prob', 1)
# hyper_params = [layer_sizes, learning_rate, mini_batch_size, pkeep]
# hyper_param_str = make_hyper_param_str(hyper_params)
layer_sizes = [4096] * 4
learning_rate = hd_exp.param('learning rate', 10**params[0])
mini_batch_size = hd_exp.param('mini batch size', int(2**params[1]))
pkeep = hd_exp.param('dropout prob', 1)
hyper_params = [layer_sizes, learning_rate, mini_batch_size, pkeep]
hyper_param_str = make_hyper_param_str(hyper_params)
# Call train function
tic = time.time()
logger.info('Start training for ' + hyper_param_str)
log_df, best_error = train(train_tuple, validation_tuple, hyper_params,
nb_epochs, random_seed, hd_exp, project_dir)
elapsed_time = time.time() - tic
logger.info('Finished training in {} s.'.format(elapsed_time))
# Writing Pandas log file to csv file on disk.
logger.info('Writing pandas DF log to disk.')
log_df.to_csv(project_dir + '/' + hyper_param_str + '/data_df.csv')
# Finish Hyperdash Experiment
hd_exp.end()
return best_error
def __init__(self, **kwargs):
self.data_augmentation = kwargs['data_augmentation']
self.epoch = kwargs['epoch']
self.gpu = (kwargs['gpu'] >= 0)
self.opt = kwargs['opt']
self.seed = kwargs['seed']
self.train = kwargs['train']
self.val = kwargs['val']
self.batchsize = kwargs['batchsize']
self.out = kwargs['out']
self.resume = kwargs['resume']
self.resume_model = kwargs['resume_model']
self.resume_opt = kwargs['resume_opt']
self.hyperdash = kwargs['hyperdash']
if self.hyperdash:
self.experiment = Experiment(self.hyperdash)
for key, val in kwargs.items():
self.experiment.param(key, val)
# validate arguments.
self._validate_arguments()
self.lowest_loss = 0
self.device = torch.device('cuda' if kwargs['gpu'] >= 0 else 'cpu')
#self.experiment.log_multiple_params(kwargs)
self.dataloader = torch.utils.data.DataLoader
def train_reconstruction_all(args):
device = torch.device(args.gpu)
df_input_data = pd.read_csv(os.path.join(
CONFIG.CSV_PATH, args.prefix + "_" + args.target_csv),
index_col=0,
encoding='utf-8-sig')
exp = Experiment(args.target_modal + " SDAE " + str(args.latent_dim),
capture_io=True)
try:
for arg, value in vars(args).items():
exp.param(arg, value)
print("Loading dataset...")
train_dataset, val_dataset = load_autoencoder_data(
df_input_data, CONFIG)
print("Loading dataset completed")
train_loader, val_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=args.shuffle), \
DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
sdae = StackedDAE(input_dim=args.input_dim,
z_dim=args.latent_dim,
binary=False,
encodeLayer=[500, 500, 2000],
decodeLayer=[2000, 500, 500],
activation="relu",
dropout=args.dropout,
device=device)
if args.resume:
print("resume from checkpoint")
sdae.load_model(
os.path.join(
CONFIG.CHECKPOINT_PATH, args.prefix + "_" +
args.target_modal + "_" + args.target_dataset + "_sdae_" +
str(args.latent_dim) + "_all.pt"))
else:
sdae.pretrain(train_loader,
val_loader,
lr=args.lr,
batch_size=args.batch_size,
num_epochs=args.pretrain_epochs,
corrupt=0.2,
loss_type="mse")
sdae.fit(train_loader,
val_loader,
lr=args.lr,
num_epochs=args.epochs,
corrupt=0.2,
loss_type="mse",
save_path=os.path.join(
CONFIG.CHECKPOINT_PATH, args.prefix + "_" +
args.target_modal + "_" + args.target_dataset + "_sdae_" +
str(args.latent_dim) + "_all.pt"))
finally:
exp.end()
def run_pusher3dof(args, sim=True, vanilla=False):
try:
from hyperdash import Experiment
hyperdash_support = True
except:
hyperdash_support = False
env = NormalizedEnv(gym.make(args.env))
torques = [1.0] * 3 # if real
colored = False
if sim:
torques = [args.t0, args.t1, args.t2]
colored = True
if not vanilla:
env.env._init(
torques=torques,
colored=colored
)
if args.seed > 0:
np.random.seed(args.seed)
env.seed(args.seed)
nb_states = env.observation_space.shape[0]
nb_actions = env.action_space.shape[0]
agent = DDPG(nb_states, nb_actions, args)
evaluate = Evaluator(
args.validate_episodes,
args.validate_steps,
args.output,
max_episode_length=args.max_episode_length
)
exp = None
if args.mode == 'train':
if hyperdash_support:
prefix = "real"
if sim: prefix = "sim"
exp = Experiment("s2r-pusher3dof-ddpg-{}".format(prefix))
import socket
exp.param("host", socket.gethostname())
exp.param("type", prefix) # sim or real
exp.param("vanilla", vanilla) # vanilla or not
exp.param("torques", torques)
exp.param("folder", args.output)
for arg in ["env", "max_episode_length", "train_iter", "seed", "resume"]:
arg_val = getattr(args, arg)
exp.param(arg, arg_val)
train(args, args.train_iter, agent, env, evaluate,
args.validate_steps, args.output,
max_episode_length=args.max_episode_length, debug=args.debug, exp=exp)
# when done
exp.end()
elif args.mode == 'test':
test(args.validate_episodes, agent, env, evaluate, args.resume,
visualize=args.vis, debug=args.debug, load_best=args.best)
else:
raise RuntimeError('undefined mode {}'.format(args.mode))
def on_train_begin(self, logs=None):
self.exp = Experiment("Deep Weather")
def train_reconstruction(args):
device = torch.device(args.gpu)
print("Loading embedding model...")
with open(
os.path.join(CONFIG.DATASET_PATH, args.target_dataset,
'word_embedding.p'), "rb") as f:
embedding_model = cPickle.load(f)
with open(os.path.join(CONFIG.DATASET_PATH, args.target_dataset,
'word_idx.json'),
"r",
encoding='utf-8') as f:
word_idx = json.load(f)
print("Loading embedding model completed")
print("Loading dataset...")
train_dataset, val_dataset = load_text_data(args,
CONFIG,
word2idx=word_idx[1])
print("Loading dataset completed")
train_loader, val_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=args.shuffle),\
DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
# t1 = max_sentence_len + 2 * (args.filter_shape - 1)
t1 = CONFIG.MAX_SENTENCE_LEN
t2 = int(math.floor(
(t1 - args.filter_shape) / 2) + 1) # "2" means stride size
t3 = int(math.floor((t2 - args.filter_shape) / 2) + 1)
args.t3 = t3
embedding = nn.Embedding.from_pretrained(
torch.FloatTensor(embedding_model))
text_encoder = text_model.ConvolutionEncoder(embedding, t3,
args.filter_size,
args.filter_shape,
args.latent_size)
text_decoder = text_model.DeconvolutionDecoder(embedding, args.tau, t3,
args.filter_size,
args.filter_shape,
args.latent_size, device)
if args.resume:
print("Restart from checkpoint")
checkpoint = torch.load(os.path.join(CONFIG.CHECKPOINT_PATH,
args.resume),
map_location=lambda storage, loc: storage)
start_epoch = checkpoint['epoch']
text_encoder.load_state_dict(checkpoint['text_encoder'])
text_decoder.load_state_dict(checkpoint['text_decoder'])
else:
print("Start from initial")
start_epoch = 0
text_autoencoder = text_model.TextAutoencoder(text_encoder, text_decoder)
criterion = nn.NLLLoss().to(device)
text_autoencoder.to(device)
optimizer = AdamW(text_autoencoder.parameters(),
lr=1.,
weight_decay=args.weight_decay,
amsgrad=True)
step_size = args.half_cycle_interval * len(train_loader)
clr = cyclical_lr(step_size,
min_lr=args.lr,
max_lr=args.lr * args.lr_factor)
scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, [clr])
if args.resume:
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
exp = Experiment("Text autoencoder " + str(args.latent_size),
capture_io=False)
for arg, value in vars(args).items():
exp.param(arg, value)
try:
text_autoencoder.train()
for epoch in range(start_epoch, args.epochs):
print("Epoch: {}".format(epoch))
for steps, batch in enumerate(train_loader):
torch.cuda.empty_cache()
feature = Variable(batch).to(device)
optimizer.zero_grad()
prob = text_autoencoder(feature)
loss = criterion(prob.transpose(1, 2), feature)
loss.backward()
optimizer.step()
scheduler.step()
if (steps * args.batch_size) % args.log_interval == 0:
input_data = feature[0]
single_data = prob[0]
_, predict_index = torch.max(single_data, 1)
input_sentence = util.transform_idx2word(
input_data.detach().cpu().numpy(),
idx2word=word_idx[0])
predict_sentence = util.transform_idx2word(
predict_index.detach().cpu().numpy(),
idx2word=word_idx[0])
print("Epoch: {} at {} lr: {}".format(
epoch, str(datetime.datetime.now()),
str(scheduler.get_lr())))
print("Steps: {}".format(steps))
print("Loss: {}".format(loss.detach().item()))
print("Input Sentence:")
print(input_sentence)
print("Output Sentence:")
print(predict_sentence)
del input_data, single_data, _, predict_index
del feature, prob, loss
exp.log("\nEpoch: {} at {} lr: {}".format(
epoch, str(datetime.datetime.now()), str(scheduler.get_lr())))
_avg_loss, _rouge_1, _rouge_2 = eval_reconstruction_with_rouge(
text_autoencoder, word_idx[0], criterion, val_loader, device)
exp.log("\nEvaluation - loss: {} Rouge1: {} Rouge2: {}".format(
_avg_loss, _rouge_1, _rouge_2))
util.save_models(
{
'epoch': epoch + 1,
'text_encoder': text_encoder.state_dict(),
'text_decoder': text_decoder.state_dict(),
'avg_loss': _avg_loss,
'Rouge1:': _rouge_1,
'Rouge2': _rouge_2,
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict()
}, CONFIG.CHECKPOINT_PATH,
"text_autoencoder_" + str(args.latent_size))
print("Finish!!!")
finally:
exp.end()
def train_reconstruction(train_loader, test_loader, encoder, decoder, args):
exp = Experiment("Reconstruction Training")
try:
lr = args.lr
encoder_opt = torch.optim.Adam(encoder.parameters(), lr=lr)
decoder_opt = torch.optim.Adam(decoder.parameters(), lr=lr)
encoder.train()
decoder.train()
steps = 0
for epoch in range(1, args.epochs+1):
print("=======Epoch========")
print(epoch)
for batch in train_loader:
feature = Variable(batch)
if args.use_cuda:
encoder.cuda()
decoder.cuda()
feature = feature.cuda()
encoder_opt.zero_grad()
decoder_opt.zero_grad()
h = encoder(feature)
prob = decoder(h)
reconstruction_loss = compute_cross_entropy(prob, feature)
reconstruction_loss.backward()
encoder_opt.step()
decoder_opt.step()
steps += 1
print("Epoch: {}".format(epoch))
print("Steps: {}".format(steps))
print("Loss: {}".format(reconstruction_loss.data[0] / args.sentence_len))
exp.metric("Loss", reconstruction_loss.data[0] / args.sentence_len)
# check reconstructed sentence
if steps % args.log_interval == 0:
print("Test!!")
input_data = feature[0]
single_data = prob[0]
_, predict_index = torch.max(single_data, 1)
input_sentence = util.transform_id2word(input_data.data, train_loader.dataset.index2word, lang="en")
predict_sentence = util.transform_id2word(predict_index.data, train_loader.dataset.index2word, lang="en")
print("Input Sentence:")
print(input_sentence)
print("Output Sentence:")
print(predict_sentence)
if steps % args.test_interval == 0:
eval_reconstruction(encoder, decoder, test_loader, args)
if epoch % args.lr_decay_interval == 0:
# decrease learning rate
lr = lr / 5
encoder_opt = torch.optim.Adam(encoder.parameters(), lr=lr)
decoder_opt = torch.optim.Adam(decoder.parameters(), lr=lr)
encoder.train()
decoder.train()
if epoch % args.save_interval == 0:
util.save_models(encoder, args.save_dir, "encoder", steps)
util.save_models(decoder, args.save_dir, "decoder", steps)
# finalization
# save vocabulary
with open("word2index", "wb") as w2i, open("index2word", "wb") as i2w:
pickle.dump(train_loader.dataset.word2index, w2i)
pickle.dump(train_loader.dataset.index2word, i2w)
# save models
util.save_models(encoder, args.save_dir, "encoder", "final")
util.save_models(decoder, args.save_dir, "decoder", "final")
print("Finish!!!")
finally:
exp.end()
# digits.py
from sklearn import svm, datasets
from hyperdash import Experiment
# Preprocess data
digits = datasets.load_digits(100)
test_cases = 50
X_train, y_train = digits.data[:-test_cases], digits.target[:-test_cases]
X_test, y_test = digits.data[-test_cases:], digits.target[-test_cases:]
# Create an experiment with a model name, then autostart
exp = Experiment("Digits Classifier")
# Record the value of hyperparameter gamma for this experiment
gamma = exp.param("gamma", 0.1)
# Param can record any basic type (Number, Boolean, String)
classifer = svm.SVC(gamma=gamma)
classifer.fit(X_train, y_train)
# Record a numerical performance metric
exp.metric("accuracy", classifer.score(X_test, y_test))
# Cleanup and mark that the experiment successfully completed
exp.end()
def __init__(self):
warnings.filterwarnings('ignore')
self.start_time = time()
self.args = get_args()
if self.args.checkpoint_dir_name:
dir_name = self.args.checkpoint_dir_name
else:
dir_name = datetime.datetime.now().strftime('%y%m%d%H%M%S')
self.path_to_dir = Path(__file__).resolve().parents[1]
self.path_to_dir = os.path.join(self.path_to_dir, *['log', dir_name])
os.makedirs(self.path_to_dir, exist_ok=True)
# tensorboard
path_to_tensorboard = os.path.join(self.path_to_dir, 'tensorboard')
os.makedirs(path_to_tensorboard, exist_ok=True)
self.writer = SummaryWriter(path_to_tensorboard)
# model saving
os.makedirs(os.path.join(self.path_to_dir, 'model'), exist_ok=True)
path_to_model = os.path.join(self.path_to_dir, *['model', 'model.tar'])
# csv
os.makedirs(os.path.join(self.path_to_dir, 'csv'), exist_ok=True)
self.path_to_results_csv = os.path.join(self.path_to_dir,
*['csv', 'results.csv'])
path_to_args_csv = os.path.join(self.path_to_dir, *['csv', 'args.csv'])
if not self.args.checkpoint_dir_name:
with open(path_to_args_csv, 'a') as f:
args_dict = vars(self.args)
param_writer = csv.DictWriter(f, list(args_dict.keys()))
param_writer.writeheader()
param_writer.writerow(args_dict)
# logging by hyperdash
if not self.args.no_hyperdash:
from hyperdash import Experiment
self.exp = Experiment('Generation task on ' + self.args.dataset +
' dataset with GAN')
for key in vars(self.args).keys():
exec("self.args.%s = self.exp.param('%s', self.args.%s)" %
(key, key, key))
else:
self.exp = None
self.dataloader = get_dataloader(self.args.dataset,
self.args.image_size,
self.args.batch_size)
sample_data = self.dataloader.__iter__().__next__()[0]
image_channels = sample_data.shape[1]
z = torch.randn(self.args.batch_size, self.args.z_dim)
self.sample_z = z.view(z.size(0), z.size(1), 1, 1)
self.Generator = Generator(self.args.z_dim, image_channels,
self.args.image_size)
self.Generator_optimizer = optim.Adam(self.Generator.parameters(),
lr=self.args.lr_Generator,
betas=(self.args.beta1,
self.args.beta2))
self.writer.add_graph(self.Generator, self.sample_z)
self.Generator.to(self.args.device)
self.Discriminator = Discriminator(image_channels,
self.args.image_size)
self.Discriminator_optimizer = optim.Adam(
self.Discriminator.parameters(),
lr=self.args.lr_Discriminator,
betas=(self.args.beta1, self.args.beta2))
self.writer.add_graph(self.Discriminator, sample_data)
self.Discriminator.to(self.args.device)
self.BCELoss = nn.BCELoss()
self.sample_z = self.sample_z.to(self.args.device)
class Experiment:
@logger.read
def __init__(self, dataset_name, model_name, optimizer_name, trial_num):
"""
:param dataset_name: name of the dataset
:type dataset_name: str
:param model_name: name of the model
:type model_name: str
:param optimizer_name: name of the optimizer
:type optimizer_name: str
:param trial_num: current number of repeated trials
:type trial_num: int
"""
# get optimized hyperparameters
with open(
f'../params/{dataset_name}_{model_name}_{optimizer_name}/result.json'
) as f:
params = json.load(f)
# get instances
self.dataset = Datasets.get(dataset_name)
self.model = Models.get(model_name, dataset=self.dataset)
self.optimizer = Optimizers.get(optimizer_name, params=params)
# get config
with open('./config.json') as f:
config = json.load(f)
# get constants
c = config['constants'][dataset_name][model_name]
self.loss = c['loss']
self.batch_size = c['batch_size']
self.epochs = c['epochs']
# configure and initialize directory
d = self.main_dir = f'../data/{dataset_name}_{model_name}_{optimizer_name}/trial{trial_num}'
if os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d)
# configure hyperdash experiment
self.hd_exp = HyperdashExperiment(
f'{dataset_name}',
api_key_getter=lambda: config['hyperdash']['api_key'])
self.hd_exp.param('dataset_name', dataset_name)
self.hd_exp.param('model_name', model_name)
self.hd_exp.param('optimizer_name', optimizer_name)
self.hd_exp.param('trial_num', trial_num)
for k, v in params.items():
self.hd_exp.param(k, v)
# set callbacks
self.callbacks = [
Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'],
self.hd_exp),
TensorBoard(log_dir=f'{self.main_dir}/tensorboard'),
TimeLogger(filename=f'{self.main_dir}/time.csv'),
CSVLogger(filename=f'{self.main_dir}/result.csv', append=True)
]
@logger.write
def begin(self):
# get data
(x_train, y_train), (x_test, y_test) = self.dataset.get_batch()
# start learning
self.model.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=['accuracy'])
self.model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
callbacks=self.callbacks,
validation_split=0.2,
verbose=2)
# save final scores
score = self.model.evaluate(x_test, y_test, verbose=1)
with open(f'{self.main_dir}/test.json', 'w') as f:
json.dump({
'test loss': score[0],
'test accuracy': score[1]
},
f,
indent=4)
# stop hyperdash experiment
self.hd_exp.end()
NUM_STEPS = 100 # episode steps
NUM_EPISODES = 1000
HIDDEN_SIZE = 128
CHKP_FREQ = 100 # model saving freq
WIDTH = 64
HEIGHT = 64
REWARD_BUF = 1000
npa = np.array
from hyperdash import Experiment
exp_name = "3DR-18-constant-shape-conv"
exp_dir = exp_name + "-" + strftime("%Y%m%d%H%M%S")
exp = Experiment(exp_name)
class Policy(nn.Module):
def __init__(self, hidden_size, num_inputs, num_outputs):
super(Policy, self).__init__()
self.relu = nn.ReLU(inplace=True)
self.conv1 = nn.Conv2d(3, 32, (3, 3), (1, 1), (1, 1))
self.conv2 = nn.Conv2d(32, 64, (3, 3), (2, 2), (1, 1))
self.linear3 = nn.Linear(64 * 32 * 32, num_outputs)
self.linear3_ = nn.Linear(64 * 32 * 32, num_outputs)
def forward(self, inputs):
x = self.relu(self.conv1(inputs))
x = self.relu(self.conv2(x))
import os
import json
import botometer
import pandas
import tweepy
import time
from subprocess import call
from tqdm import tqdm
from hyperdash import Experiment
import subprocess
import pickle
import copy
import sys
import rotem_helpers
exp = Experiment("bot_meter")
version = 3
users_follow = {}
# rapidapi_key = "OyTByfzOA2mshbg7TNfI9gxuqleyp1Ne1RXjsni94N9JOht0ZB" # now it's called rapidapi key
rapidapi_key = '8a8d64e8b5msh34c092335ddc0b0p125cb8jsne6b7f9d11cea'
twitter_app_auth = {
'consumer_key': 'XZqb7nIARNbh3x4KxaInQ',
'consumer_secret': 'MHYtjLH6CqekMxR8sQtH6trnEXfdNCMvd75Dv5akWo',
'access_token': '245305900-NTgpfmVo4XK39SCwhBZ10SuWEnj1MRu0ymv2h6CJ',
'access_token_secret': 'XYyP5fG4tQL3chz6p7x71pjTi883CJA59g72Bran1bC2P',
}
bom = botometer.Botometer(wait_on_ratelimit=True,
rapidapi_key=rapidapi_key,
# checkpoint = torch.load(MODEL_PATH_BEST)
# net.load_state_dict(checkpoint['state_dict'])
# print("MODEL LOADED, CONTINUING TRAINING")
# return "TRAINING AVG LOSS: {}\n" \
# "TRAINING AVG DIFF: {}".format(
# checkpoint["epoch_avg_loss"], checkpoint["epoch_avg_diff"])
# else:
# if optional:
# pass # model loading was optional, so nothing to do
# else:
# # shit, no model
# raise Exception("model couldn't be found:", MODEL_PATH_BEST)
loss_function = nn.MSELoss()
if hyperdash_support:
exp = Experiment("[sim2real] lstm - real v3")
exp.param("exp", EXPERIMENT)
exp.param("layers", LSTM_LAYERS)
exp.param("nodes", HIDDEN_NODES)
# if TRAIN:
optimizer = optim.Adam(net.parameters())
# if CONTINUE:
# old_model_string = loadModel(optional=True)
# print(old_model_string)
# else:
# old_model_string = loadModel(optional=False)
loss_history = [np.inf] # very high loss because loss can't be empty for min()
for epoch in np.arange(EPOCHS):
import numpy as np
from hyperdash import Experiment
from agent.trainer import Trainer
from agent.util import EpsilonExponentialDecay
from marlenv.goldmine.relative import GoldmineRV
from marlenv.util import GoldmineRecorder
from agent.deepq.simple_dqn import SimpleDQN
name = 'gv_n4'
exp = Experiment(name)
agent_num = 6
task_num = 4
view_range = 3
env = GoldmineRV(agent_num, task_num, view_range)
env.seed(0)
obs_num = 2
observation_space = env.observation_space[0:2] + (env.observation_space[2] *
obs_num, )
def preprocess(obs):
n = len(obs)
pr_obs = np.empty((n, ) + observation_space)
for i, o in enumerate(obs):
pr_obs[i] = np.dstack(o)
return pr_obs
params = {
'name':
def gen_estimator(period=None):
resnet_size = int(flags_obj.resnet_size)
data_format = flags_obj.data_format
batch_size = flags_obj.batch_size
resnet_version = int(flags_obj.resnet_version)
loss_scale = flags_core.get_loss_scale(flags_obj)
dtype_tf = flags_core.get_tf_dtype(flags_obj)
num_epochs_per_decay = flags_obj.num_epochs_per_decay
learning_rate_decay_factor = flags_obj.learning_rate_decay_factor
end_learning_rate = flags_obj.end_learning_rate
learning_rate_decay_type = flags_obj.learning_rate_decay_type
weight_decay = flags_obj.weight_decay
zero_gamma = flags_obj.zero_gamma
lr_warmup_epochs = flags_obj.lr_warmup_epochs
base_learning_rate = flags_obj.base_learning_rate
use_resnet_d = flags_obj.use_resnet_d
use_dropblock = flags_obj.use_dropblock
dropblock_kp = [float(be) for be in flags_obj.dropblock_kp]
label_smoothing = flags_obj.label_smoothing
momentum = flags_obj.momentum
bn_momentum = flags_obj.bn_momentum
train_epochs = flags_obj.train_epochs
piecewise_lr_boundary_epochs = [
int(be) for be in flags_obj.piecewise_lr_boundary_epochs
]
piecewise_lr_decay_rates = [
float(dr) for dr in flags_obj.piecewise_lr_decay_rates
]
use_ranking_loss = flags_obj.use_ranking_loss
use_se_block = flags_obj.use_se_block
use_sk_block = flags_obj.use_sk_block
mixup_type = flags_obj.mixup_type
dataset_name = flags_obj.dataset_name
kd_temp = flags_obj.kd_temp
no_downsample = flags_obj.no_downsample
anti_alias_filter_size = flags_obj.anti_alias_filter_size
anti_alias_type = flags_obj.anti_alias_type
cls_loss_type = flags_obj.cls_loss_type
logit_type = flags_obj.logit_type
embedding_size = flags_obj.embedding_size
pool_type = flags_obj.pool_type
arc_s = flags_obj.arc_s
arc_m = flags_obj.arc_m
bl_alpha = flags_obj.bl_alpha
bl_beta = flags_obj.bl_beta
exp = None
if install_hyperdash and flags_obj.use_hyperdash:
exp = Experiment(flags_obj.model_dir.split("/")[-1])
resnet_size = exp.param("resnet_size", int(flags_obj.resnet_size))
batch_size = exp.param("batch_size", flags_obj.batch_size)
exp.param("dtype", flags_obj.dtype)
learning_rate_decay_type = exp.param(
"learning_rate_decay_type", flags_obj.learning_rate_decay_type)
weight_decay = exp.param("weight_decay", flags_obj.weight_decay)
zero_gamma = exp.param("zero_gamma", flags_obj.zero_gamma)
lr_warmup_epochs = exp.param("lr_warmup_epochs",
flags_obj.lr_warmup_epochs)
base_learning_rate = exp.param("base_learning_rate",
flags_obj.base_learning_rate)
use_dropblock = exp.param("use_dropblock", flags_obj.use_dropblock)
dropblock_kp = exp.param(
"dropblock_kp", [float(be) for be in flags_obj.dropblock_kp])
piecewise_lr_boundary_epochs = exp.param(
"piecewise_lr_boundary_epochs",
[int(be) for be in flags_obj.piecewise_lr_boundary_epochs])
piecewise_lr_decay_rates = exp.param(
"piecewise_lr_decay_rates",
[float(dr) for dr in flags_obj.piecewise_lr_decay_rates])
mixup_type = exp.param("mixup_type", flags_obj.mixup_type)
dataset_name = exp.param("dataset_name", flags_obj.dataset_name)
exp.param("autoaugment_type", flags_obj.autoaugment_type)
classifier = tf.estimator.Estimator(
model_fn=model_function,
model_dir=flags_obj.model_dir,
config=run_config,
params={
'resnet_size': resnet_size,
'data_format': data_format,
'batch_size': batch_size,
'resnet_version': resnet_version,
'loss_scale': loss_scale,
'dtype': dtype_tf,
'num_epochs_per_decay': num_epochs_per_decay,
'learning_rate_decay_factor': learning_rate_decay_factor,
'end_learning_rate': end_learning_rate,
'learning_rate_decay_type': learning_rate_decay_type,
'weight_decay': weight_decay,
'zero_gamma': zero_gamma,
'lr_warmup_epochs': lr_warmup_epochs,
'base_learning_rate': base_learning_rate,
'use_resnet_d': use_resnet_d,
'use_dropblock': use_dropblock,
'dropblock_kp': dropblock_kp,
'label_smoothing': label_smoothing,
'momentum': momentum,
'bn_momentum': bn_momentum,
'embedding_size': embedding_size,
'train_epochs': train_epochs,
'piecewise_lr_boundary_epochs': piecewise_lr_boundary_epochs,
'piecewise_lr_decay_rates': piecewise_lr_decay_rates,
'with_drawing_bbox': flags_obj.with_drawing_bbox,
'use_ranking_loss': use_ranking_loss,
'use_se_block': use_se_block,
'use_sk_block': use_sk_block,
'mixup_type': mixup_type,
'kd_temp': kd_temp,
'no_downsample': no_downsample,
'dataset_name': dataset_name,
'anti_alias_filter_size': anti_alias_filter_size,
'anti_alias_type': anti_alias_type,
'cls_loss_type': cls_loss_type,
'logit_type': logit_type,
'arc_s': arc_s,
'arc_m': arc_m,
'pool_type': pool_type,
'bl_alpha': bl_alpha,
'bl_beta': bl_beta,
'train_steps': total_train_steps,
})
return classifier, exp
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer = 'Nadam',
metrics=['accuracy'])
'''
#start
exp = Experiment("Experiment 1")
input_img = Input(shape=(250, 250, 3))
#input_img2 = Input(shape=(250, 250, 3))
tower_1 = Conv2D(42, (3, 3), activation='relu', padding='same')(input_img)
#tower_1 = BatchNormalization()(tower_1)
tower_1 = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(tower_1)
#tower_1 = BatchNormalization()(tower_1)
#tower_1 = Dropout(.2)(tower_1)
tower_1 = Conv2D(74, (3, 3), activation='relu', padding='same')(tower_1)
#tower_1 = Conv2D(64, (3, 3), padding='same', activation='relu')(tower_1)
tower_1 = BatchNormalization()(tower_1)
tower_1 = MaxPooling2D((2, 2), strides=(2, 2), padding='same')(tower_1)
#tower_1 = BatchNormalization()(tower_1)
return x
def to_var(x, volatile=False):
x = Variable(torch.from_numpy(x), volatile=volatile)
if torch.cuda.is_available():
x = x.cuda()
return x
def save_model(state):
torch.save({"state_dict": state}, MODEL_PATH)
loss_function = torch.nn.MSELoss()
exp = Experiment("[sim2real] lstm-realv6")
# exp.param("exp", EXPERIMENT)
# exp.param("layers", LSTM_LAYERS)
# exp.param("nodes", HIDDEN_NODES)
optimizer = torch.optim.Adam(net.parameters())
robot = SingleRobot(debug=False)
for epoch in range(EPOCHS):
for epi in range(len(ds.current_real)):
net.zero_hidden() # !important
net.hidden[0].detach_() # !important
net.hidden[1].detach_() # !important
net.zero_grad()
optimizer.zero_grad()
checkpoint = torch.load(MODEL_PATH_BEST)
net.load_state_dict(checkpoint['state_dict'])
return "TRAINING AVG LOSS: {}\n" \
"TRAINING AVG DIFF: {}".format(
checkpoint["epoch_avg_loss"], checkpoint["epoch_avg_diff"])
else:
if optional:
pass # model loading was optional, so nothing to do
else:
#shit, no model
raise Exception("model couldn't be found:", MODEL_PATH_BEST)
loss_function = nn.MSELoss()
if hyperdash_support:
exp = Experiment("simple lstm - fl4")
exp.param("layers", LSTM_LAYERS)
exp.param("nodes", HIDDEN_NODES)
if TRAIN:
optimizer = optim.Adam(net.parameters())
if CONTINUE:
old_model_string = loadModel(optional=True)
print(old_model_string)
else:
old_model_string = loadModel(optional=False)
loss_history = [9999999
] # very high loss because loss can't be empty for min()
# h0 = Variable(torch.randn(, 3, 20))
# c0 = Variable(torch.randn(2, 3, 20))
class KerasModel:
def __init__(self,
name='deathbot',
load_weights=False,
training=False,
batch_size=100,
lr=1e-3,
location=None):
self.session = tf.Session()
self.name = name
if training:
from hyperdash import Experiment
self.exp = Experiment(name)
if name in MODELS.keys():
self.model = MODELS[name]() if not training else MODELS[name](
self.exp)
adam = Adam(lr=lr)
nadam = Nadam(lr=lr)
#rms = RMSprop(lr=lr)
#sgd = SGD(lr=lr)
self.optimizer = adam if name == "evo" else nadam
loss = ["binary_crossentropy", "categorical_crossentropy", "poisson"]
self.model.compile(optimizer=self.optimizer,
loss=loss[1],
metrics=["acc"])
self.callbacks = []
if training:
self.exp.param("lr", lr)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
min_lr=1e-4,
verbose=1)
tb = TensorBoard('./models/logs/{}'.format(name), write_graph=True)
cp = ModelCheckpoint(
filepath='./models/weights-{}.hdf5'.format(name),
monitor='val_acc',
verbose=1,
save_best_only=True)
hd = Hyperdash(self.exp, self.model)
es = EarlyStopping('val_acc', patience=5, verbose=1)
self.callbacks = [cp, tb, hd, reduce_lr, es]
if load_weights:
#print(os.listdir(os.getcwd()))
self.model.load_weights('./deathbot/weights-{}.hdf5'.format(name))
if training: print('Weights Loaded...')
def save(self, path):
self.model.save(path + self.name + ".h5")
def fit(self, input_data, expected_output_data, batch_size=100, epochs=1):
input_data = self.normalize_input(input_data)
return self.model.fit(input_data,
expected_output_data,
batch_size=batch_size,
epochs=epochs,
verbose=1,
callbacks=self.callbacks,
validation_split=0.2,
shuffle=False)
def predict(self, input_data, batch_size=1, p=False):
return list(
map(
self.clean_pred,
self.model.predict(self.normalize_input(np.array([input_data
])),
batch_size=batch_size)[0]))
def compute_loss(self, input_data, expected_output_data):
return self.model.evaluate(self.normalize_input(input_data),
expected_output_data,
batch_size=1,
verbose=1)
@staticmethod
def clean_pred(pred):
return pred if pred > 0.01 else 0.0
@staticmethod
def normalize_input(input_data):
# Assert the shape is what we expect
assert len(input_data.shape) == 3 and input_data.shape[
1] == PLANET_MAX_NUM and input_data.shape[2] == PER_PLANET_FEATURES
m = np.expand_dims(input_data.mean(axis=1), axis=1)
s = np.expand_dims(input_data.std(axis=1), axis=1)
return (input_data - m) / (s + 1e-6)
class GAN(object):
def __init__(self):
warnings.filterwarnings('ignore')
self.start_time = time()
self.args = get_args()
if self.args.checkpoint_dir_name:
dir_name = self.args.checkpoint_dir_name
else:
dir_name = datetime.datetime.now().strftime('%y%m%d%H%M%S')
self.path_to_dir = Path(__file__).resolve().parents[1]
self.path_to_dir = os.path.join(self.path_to_dir, *['log', dir_name])
os.makedirs(self.path_to_dir, exist_ok=True)
# tensorboard
path_to_tensorboard = os.path.join(self.path_to_dir, 'tensorboard')
os.makedirs(path_to_tensorboard, exist_ok=True)
self.writer = SummaryWriter(path_to_tensorboard)
# model saving
os.makedirs(os.path.join(self.path_to_dir, 'model'), exist_ok=True)
path_to_model = os.path.join(self.path_to_dir, *['model', 'model.tar'])
# csv
os.makedirs(os.path.join(self.path_to_dir, 'csv'), exist_ok=True)
self.path_to_results_csv = os.path.join(self.path_to_dir,
*['csv', 'results.csv'])
path_to_args_csv = os.path.join(self.path_to_dir, *['csv', 'args.csv'])
if not self.args.checkpoint_dir_name:
with open(path_to_args_csv, 'a') as f:
args_dict = vars(self.args)
param_writer = csv.DictWriter(f, list(args_dict.keys()))
param_writer.writeheader()
param_writer.writerow(args_dict)
# logging by hyperdash
if not self.args.no_hyperdash:
from hyperdash import Experiment
self.exp = Experiment('Generation task on ' + self.args.dataset +
' dataset with GAN')
for key in vars(self.args).keys():
exec("self.args.%s = self.exp.param('%s', self.args.%s)" %
(key, key, key))
else:
self.exp = None
self.dataloader = get_dataloader(self.args.dataset,
self.args.image_size,
self.args.batch_size)
sample_data = self.dataloader.__iter__().__next__()[0]
image_channels = sample_data.shape[1]
z = torch.randn(self.args.batch_size, self.args.z_dim)
self.sample_z = z.view(z.size(0), z.size(1), 1, 1)
self.Generator = Generator(self.args.z_dim, image_channels,
self.args.image_size)
self.Generator_optimizer = optim.Adam(self.Generator.parameters(),
lr=self.args.lr_Generator,
betas=(self.args.beta1,
self.args.beta2))
self.writer.add_graph(self.Generator, self.sample_z)
self.Generator.to(self.args.device)
self.Discriminator = Discriminator(image_channels,
self.args.image_size)
self.Discriminator_optimizer = optim.Adam(
self.Discriminator.parameters(),
lr=self.args.lr_Discriminator,
betas=(self.args.beta1, self.args.beta2))
self.writer.add_graph(self.Discriminator, sample_data)
self.Discriminator.to(self.args.device)
self.BCELoss = nn.BCELoss()
self.sample_z = self.sample_z.to(self.args.device)
def train(self):
self.train_hist = {}
self.train_hist['Generator_loss'] = 0.0
self.train_hist['Discriminator_loss'] = 0.0
# real ---> y = 1
# fake ---> y = 0
self.y_real = torch.ones(self.args.batch_size, 1).to(self.args.device)
self.y_fake = torch.zeros(self.args.batch_size, 1).to(self.args.device)
self.Discriminator.train()
global_step = 0
# -----training -----
for epoch in range(1, self.args.n_epoch + 1):
self.Generator.train()
for idx, (x, _) in enumerate(self.dataloader):
if idx == self.dataloader.dataset.__len__(
) // self.args.batch_size:
break
z = torch.randn(self.args.batch_size, self.args.z_dim)
z = z.view(z.size(0), z.size(1), 1, 1)
z = z.to(self.args.device)
x = x.to(self.args.device)
# ----- update Discriminator -----
# minimize: -{ log[D(x)] + log[1-D(G(z))] }
self.Discriminator_optimizer.zero_grad()
# real
# ---> log[D(x)]
Discriminator_real, _ = self.Discriminator(x)
Discriminator_real_loss = self.BCELoss(Discriminator_real,
self.y_real)
# fake
# ---> log[1-D(G(z))]
Discriminator_fake, _ = self.Discriminator(self.Generator(z))
Discriminator_fake_loss = self.BCELoss(Discriminator_fake,
self.y_fake)
Discriminator_loss = Discriminator_real_loss + Discriminator_fake_loss
self.train_hist[
'Discriminator_loss'] = Discriminator_loss.item()
Discriminator_loss.backward()
self.Discriminator_optimizer.step()
# ----- update Generator -----
# As stated in the original paper,
# we want to train the Generator
# by minimizing log(1−D(G(z)))
# in an effort to generate better fakes.
# As mentioned, this was shown by Goodfellow
# to not provide sufficient gradients,
# especially early in the learning process.
# As a fix, we instead wish to maximize log(D(G(z))).
# ---> minimize: -log[D(G(z))]
self.Generator_optimizer.zero_grad()
Discriminator_fake, _ = self.Discriminator(self.Generator(z))
Generator_loss = self.BCELoss(Discriminator_fake, self.y_real)
self.train_hist['Generator_loss'] = Generator_loss.item()
Generator_loss.backward()
self.Generator_optimizer.step()
# ----- logging by tensorboard, csv and hyperdash
# tensorboard
self.writer.add_scalar('loss/Generator_loss',
Generator_loss.item(), global_step)
self.writer.add_scalar('loss/Discriminator_loss',
Discriminator_loss.item(), global_step)
# csv
with open(self.path_to_results_csv, 'a') as f:
result_writer = csv.DictWriter(
f, list(self.train_hist.keys()))
if epoch == 1 and idx == 0: result_writer.writeheader()
result_writer.writerow(self.train_hist)
# hyperdash
if self.exp:
self.exp.metric('Generator loss', Generator_loss.item())
self.exp.metric('Discriminator loss',
Discriminator_loss.item())
if (idx % 10) == 0:
self._plot_sample(global_step)
global_step += 1
elapsed_time = time() - self.start_time
print('\nTraining Finish, elapsed time ---> %f' % (elapsed_time))
def _plot_sample(self, global_step):
with torch.no_grad():
total_n_sample = min(self.args.n_sample, self.args.batch_size)
image_frame_dim = int(np.floor(np.sqrt(total_n_sample)))
samples = self.Generator(self.sample_z)
samples = samples.cpu().data.numpy().transpose(0, 2, 3, 1)
samples = (samples + 1) / 2
fig = plt.figure(figsize=(24, 15))
for i in range(image_frame_dim * image_frame_dim):
ax = fig.add_subplot(
image_frame_dim,
image_frame_dim * 2,
(int(i / image_frame_dim) + 1) * image_frame_dim + i + 1,
xticks=[],
yticks=[])
if samples[i].shape[2] == 3:
ax.imshow(samples[i])
else:
ax.imshow(samples[i][:, :, 0], cmap='gray')
self.writer.add_figure('sample images generated by GAN', fig,
global_step)
