def startExperiment(self):
experiment = Experiment(api_key=self.API_KEY,
project_name=self.comet_project_name,
workspace=self.WORKSPACE,
log_code=True)
# Disable cuda support
if not self.cuda_support:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
print(self.experiment_name)
features_temp = []
data, class_labels = DataGrabber.getAudio5Sec()
if self.vst_parameter is not None:
for i, label in enumerate(class_labels):
new_paras = []
for para in self.vst_parameter:
new_paras.append(label[para])
class_labels[i] = new_paras
features_temp.append([data, class_labels])
model = self.prepare_data_and_train(features_temp)
del features_temp
experiment.end()
def train(self):
os.mkdir(self.paths['path'])
if self.use_comet and self.api_key and self.project_name and self.workspace:
experiment = Experiment(api_key=self.api_key,
project_name=self.project_name,
workspace=self.workspace)
experiment.log_dataset_hash(self.train_dataset)
experiment.add_tags([
str(self.architecture), "text_generation",
f"nb_labels_{self.number_labels}"
])
with experiment.train():
hist = self.fit_dataset(self.train_dataset, self.val_dataset,
self.epochs)
experiment.end()
elif self.use_comet:
raise Exception(
"Please provide an api_key, project_name and workspace for comet_ml"
)
else:
callbacks = self.callback_func(
tensorboard_dir=self.paths['tensorboard_path'],
checkpoint_path=self.paths['checkpoint_path'])
hist = self.model.fit_dataset(self.train_dataset, self.val_dataset,
self.epochs, callbacks)
self.metrics = get_metrics(hist, "sparse_categorical_accuracy")
self.export_weights(self.model)
self.export_info(self.model_info)
self.export_metrics(self.metrics)
self.export_tokenizer(self.tokenizer)
if self.do_zip_model:
self.zip_model()
def pretrain(self, save_filename, patience):
''' Train & validate the model on full training dataset (all pilots) from scratch.
Inputs:
- save_filename: (string: 'weights_save_filename.h5').
- patience: Number of epochs without improvement before training stops (int).
Output:
- hist: Contains loss, acc, val_loss, val_acc metrics over epochs. (dictionnary)
'''
X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], []
if self.log_pilots:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='All pilots', log_code=False, auto_param_logging=False)
for pilot_idx in range(1, self.n_pilots + 1):
X_train_pilot, y_train_pilot, X_valid_pilot, y_valid_pilot, X_test_pilot, y_test_pilot = self.load_data(pilot_idx, valid_ratio=0.2)
# Stacking training/validation datasets of each pilot into a big dataset
if len(X_train)==0 and len(X_valid)==0:
X_train = X_train_pilot
y_train = y_train_pilot
X_valid = X_valid_pilot
y_valid = y_valid_pilot
X_test = X_test_pilot
y_test = y_test_pilot
else:
X_train = np.concatenate([X_train, X_train_pilot], axis=0)
y_train = np.concatenate([y_train, y_train_pilot], axis=0)
X_valid = np.concatenate([X_valid, X_valid_pilot], axis=0)
y_valid = np.concatenate([y_valid, y_valid_pilot], axis=0)
X_test = np.concatenate([X_test, X_test_pilot], axis=0)
y_test = np.concatenate([y_test, y_test_pilot], axis=0)
# Shuffle
X_train, y_train = shuffle(X_train, y_train, random_state=0)
X_valid, y_valid = shuffle(X_valid, y_valid, random_state=0)
X_test, y_test = shuffle(X_test, y_test, random_state=0)
# Build model
self.model = self.build_model(load_weights=False)
# Train on all pilots
hist, _ = self.train(X_train, y_train, X_valid, y_valid, save_filename, patience)
# Get some metrics
score = self.test('all', save_filename, X_train, y_train, X_test, y_test, False)
if self.log_pilots:
experiment.log_metrics({'Test accuracy' : score})
experiment.end()
return hist
class CometMLMonitor(MonitorBase):
"""
Send data to https://www.comet.ml.
Note:
1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack.
2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze.
Therefore the feature is disabled by default.
"""
def __init__(self, experiment=None, api_key=None, tags=None, **kwargs):
"""
Args:
experiment (comet_ml.Experiment): if provided, invalidate all other arguments
api_key (str): your comet.ml API key
tags (list[str]): experiment tags
kwargs: other arguments passed to :class:`comet_ml.Experiment`.
"""
if experiment is not None:
self._exp = experiment
assert api_key is None and tags is None and len(kwargs) == 0
else:
from comet_ml import Experiment
kwargs.setdefault(
'log_code', True
) # though it's not functioning, git patch logging requires it
kwargs.setdefault('auto_output_logging', None)
self._exp = Experiment(api_key=api_key, **kwargs)
if tags is not None:
self._exp.add_tags(tags)
self._exp.set_code(
"Code logging is impossible because there are too many files ...")
self._exp.log_dependency('tensorpack', __git_version__)
@property
def experiment(self):
"""
The :class:`comet_ml.Experiment` instance.
"""
return self._exp
def _before_train(self):
self._exp.set_model_graph(tf.get_default_graph())
@HIDE_DOC
def process_scalar(self, name, val):
self._exp.log_metric(name, val, step=self.global_step)
def _after_train(self):
self._exp.end()
def _after_epoch(self):
self._exp.log_epoch_end(self.epoch_num)
def run(gpu_id, world_size, args):
"""
This is a single process that is linked to a single GPU
:param gpu_id: The id of the GPU on the current node
:param world_size: Total number of processes across nodes
:param args:
:return:
"""
torch.cuda.set_device(gpu_id)
# The overall rank of this GPU process across multiple nodes
global_process_rank = args.nr * args.gpus + gpu_id
if global_process_rank == 0:
experiment = Experiment(auto_output_logging="simple")
else:
experiment = Experiment(disabled=True)
print(
f"Running DDP model on Global Process with Rank: {global_process_rank }."
)
setup(global_process_rank, world_size, args.backend)
model = build_model()
model.cuda(gpu_id)
ddp_model = DDP(model, device_ids=[gpu_id])
criterion = nn.CrossEntropyLoss().cuda(gpu_id)
optimizer = optim.Adam(model.parameters())
# Load training data
train_dataset = torchvision.datasets.MNIST(root="./data",
train=True,
transform=transforms.ToTensor(),
download=True)
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset, num_replicas=world_size, rank=global_process_rank)
trainloader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=BATCH_SIZE,
shuffle=False,
num_workers=0,
pin_memory=True,
sampler=train_sampler,
)
for epoch in range(1, args.epochs + 1):
train(ddp_model, optimizer, criterion, trainloader, epoch)
cleanup()
experiment.end()
def _train_with_comet(self, train_dataset, val_dataset):
experiment = Experiment(api_key=self.api_key,
project_name=self.project_name,
workspace=self.workspace)
experiment.log_dataset_hash(train_dataset)
experiment.add_tags([
str(self.architecture), self.name,
f"nb_labels_{self.label_encoder_classes_number}"
])
with experiment.train():
hist = self.fit_dataset(train_dataset, val_dataset)
experiment.end()
return hist
def startExperiment(self):
experiment = Experiment(api_key=self.API_KEY, project_name=self.comet_project_name, workspace=self.WORKSPACE,
log_code=True)
# Disable cuda support
if not self.cuda_support:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
print(self.experiment_name)
features_temp = []
# for audio
# data, class_labels = DataGrabber.getAudio5Sec()
# if self.vst_parameter is not None:
# for i, label in enumerate(class_labels):
# new_paras = []
# for para in self.vst_parameter:
# new_paras.append(label[para])
# class_labels[i] = new_paras
self.features = DataGrabber.getZeroCrossingRate256_64()
dir_files = os.listdir(self.DATASET_DIRECTORY_PATH)
dir_files.sort(key=lambda f: int(re.sub('\D', '', f)))
para_count = 0
for file in dir_files:
if "xml" in file:
sample_number = int(file.split(".")[0][4:])
tree = ET.parse(os.path.join(os.path.abspath(self.DATASET_DIRECTORY_PATH), file))
root = tree.getroot()
class_labels = []
if self.vst_parameter is None:
for x in range(99):
class_labels.append(int(root[x + 2].attrib["presetValue"]))
para_count = 99
else:
para_count = len(self.vst_parameter)
for para in self.vst_parameter:
class_labels.append(int(root[para + 2].attrib["presetValue"]))
data = self.features[sample_number]
features_temp.append([data, class_labels])
del dir_files, self.features, tree, root
print("All Data Appended")
model = self.prepare_data_and_train(features_temp)
del features_temp
experiment.end()
class CometLogger(LightningLoggerBase):
def __init__(self, *args, **kwargs):
super(CometLogger, self).__init__()
self.experiment = CometExperiment(*args, **kwargs)
@rank_zero_only
def log_hyperparams(self, params):
self.experiment.log_parameters(vars(params))
@rank_zero_only
def log_metrics(self, metrics, step_num):
# self.experiment.set_epoch(self, metrics.get('epoch', 0))
self.experiment.log_metrics(metrics)
@rank_zero_only
def finalize(self, status):
self.experiment.end()
def training_loop(x,
y,
cfg,
exp_name="exp_1",
device=torch.device('cuda'),
starting_epoch=1):
model = init_model(cfg, device)
datasets = init_datasets(x, y, cfg)
dataloaders = init_loaders(datasets, cfg)
optim = init_optimizers(cfg, model)
n_epochs = cfg.TRAIN.MAX_EPOCHS
experiment = Experiment(project_name='mbm-pos-metrics',
api_key='w4JbvdIWlas52xdwict9MwmyH')
experiment.set_name(exp_name)
hyper = {
'train_batch_size': cfg.TRAIN.BATCH_SIZE,
'age_lambda': cfg.MODEL.AGE_LAMBDA,
'n_epochs': cfg.TRAIN.MAX_EPOCHS,
'learning_rate': cfg.TRAIN.LEARNING_RATE,
'weight_decay': cfg.MODEL.WEIGHT_DECAY
}
experiment.log_parameters(hyper)
print("Starting training loop!")
for epoch in range(starting_epoch, n_epochs + 1):
for phase in ['train', 'val', 'test']:
print("Epoch {}/{} - {}".format(epoch, n_epochs, phase.upper()))
forward_model(model,
dataloaders[phase],
optim,
cfg,
device,
experiment,
phase,
current_epoch=epoch)
# checkpoint model
check = {
'model': model.state_dict(),
'optim': optim['optimizer'].state_dict()
}
torch.save(
check,
"../../storage/mbm/checkpoints/{}_{}.pth".format(exp_name, epoch))
experiment.end()
return experiment
class Logger:
def __init__(self, send_logs, tags, parameters):
self.send_logs = send_logs
if self.send_logs:
self.experiment = Experiment(api_key="OZwyhJHyqzPZgHEpDFL1zxhyI",
project_name="drilling-the-hole",
workspace="wwydmanski")
self.sent_mb = 0
if self.send_logs:
if tags is not None:
self.experiment.add_tags(tags)
if parameters is not None:
self.experiment.log_parameters(parameters)
def begin_logging(self, episode_count, steps_per_ep):
if self.send_logs:
self.experiment.log_parameter("Episode count", episode_count)
self.experiment.log_parameter("Max steps per episode",
steps_per_ep)
def log_round(self, actions, reward, cumulative_reward, angle, loss, step):
if self.send_logs:
self.experiment.log_metric("Round reward", reward, step=step)
self.experiment.log_metric("Per-ep reward",
cumulative_reward,
step=step)
self.experiment.log_metric("Action 1", actions[0], step=step)
self.experiment.log_metric("Action 2", actions[1], step=step)
self.experiment.log_metric("Current angle", angle, step=step)
self.experiment.log_metrics(loss, step=step)
def log_episode(self, cumulative_reward, state, step):
if self.send_logs:
self.experiment.log_metric("Angle", state[0], step=step)
self.experiment.log_metric("Goal", state[1], step=step)
self.experiment.log_metric("Cumulative reward",
cumulative_reward,
step=step)
def end(self):
if self.send_logs:
self.experiment.end()
def log_hyperparameters_to_comet(clf, experiment):
for i in range(len(clf.cv_results_["params"])):
exp = Experiment(
workspace="s0lvang",
project_name="ideal-pancake-hyperparameter",
api_key=globals.flags.comet_api_key,
)
exp.add_tag("hp_tuning")
exp.add_tags(globals.comet_logger.get_tags())
for k, v in clf.cv_results_.items():
if k == "params":
exp.log_parameters(v[i])
else:
exp.log_metric(k, v[i])
exp.end()
old_experiment = ExistingExperiment(
api_key=globals.flags.comet_api_key,
previous_experiment=experiment.get_key(),
)
globals.comet_logger = old_experiment
def test_comet(self):
"""Test with a comet hook."""
from comet_ml import Experiment
comet = Experiment(project_name="Testing",
auto_output_logging="native")
comet.log_dataset_info(name="Karcher", path="shonan")
comet.add_tag("GaussNewton")
comet.log_parameter("method", "GaussNewton")
time = datetime.now()
comet.set_name("GaussNewton-" + str(time.month) + "/" + str(time.day) +
" " + str(time.hour) + ":" + str(time.minute) + ":" +
str(time.second))
# I want to do some comet thing here
def hook(optimizer, error):
comet.log_metric("Karcher error", error, optimizer.iterations())
gtsam_optimize(self.optimizer, self.params, hook)
comet.end()
actual = self.optimizer.values()
self.gtsamAssertEquals(actual.atRot3(KEY), self.expected)
class CometLogger():
def __init__(self):
global comet_installed
self._logging = False
if comet_installed:
try:
self._experiment = Experiment(auto_metric_logging=False,
display_summary_level=0)
self._experiment.log_other("Created from", "sweetviz!")
self._logging = True
except:
print(
"ERROR: comet_ml is installed, but not configured properly (e.g. check API key setup). HTML reports will not be uploaded."
)
def log_html(self, html_content):
if self._logging:
try:
self._experiment.log_html(html_content)
except:
print(
"comet_ml.log_html(): error occurred during call to log_html()."
)
else:
print(
"comet_ml.log_html(): comet_ml is not installed or otherwise ready for logging."
)
def end(self):
if self._logging:
try:
self._experiment.end()
except:
print("comet_ml.end(): error occurred during call to end().")
else:
print(
"comet_ml.end(): comet_ml is not installed or otherwise ready."
)
def log(self, experiment=None):
''' Export all logs in the Comet.ml environment.
See https://www.comet.ml/ for more details
'''
# Initialize Comet.ml experience (naming, tags) for automatic logging
project_name = 'Optimization' if self.comet_optimize else 'Summary'
experiment_name = '{} - {} '.format(self.model_name, str(self.batch_size)) + ('ES+' if self.train_after_es else '')
experiment_tags = [ self.model_name, self.monitor_val ] + (['ES+'] if self.train_after_es else []) + (['Pre-train'] if self.pretraining else [])
if experiment == None:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name=project_name, log_code=False, auto_param_logging=False, auto_metric_logging=False)
experiment.set_name(experiment_name)
experiment.add_tags(experiment_tags)
# Export hyperparameters
experiment.log_parameters(self.dataloader_params)
experiment.log_parameters(self.training_params)
# Export metrics values
experiment.log_metrics({'Average accuracy' : np.mean(self.test_score['accuracy']), 'Std accuracy' : np.std(self.test_score['accuracy'])})
# Export metrics graphs for each pilot (accuracy, loss, confusion matrix)
[ experiment.log_figure(figure_name='Confusion matrix {}'.format(pilot_idx), figure=plot_cm(self.conf_matrices, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
[ experiment.log_figure(figure_name='Loss pilot {}'.format(pilot_idx), figure=plot_loss(self.histories[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
fig, ax = plt.subplots(figsize=(10,6))
plot_full_barchart(self.test_score, n_pilots=self.n_pilots, title=' {} ConvNet model'.format(self.model_name), fig=fig)
experiment.log_figure(figure_name='Accuracy barchart', figure=fig)
if self.train_after_es:
[ experiment.log_figure(figure_name='Loss pilot {} (ES+)'.format(pilot_idx), figure=plot_loss(self.histories_es[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
# Export model weights for each pilot
[ experiment.log_asset('{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)]
experiment.end()
def main(args):
train_set = KBDataset(args.train_ratings, args.prefetch_to_gpu)
test_set = KBDataset(args.test_ratings, args.prefetch_to_gpu)
train_fairness_set = NodeClassification(args.users_train,
args.prefetch_to_gpu)
test_fairness_set = NodeClassification(args.users_test,
args.prefetch_to_gpu)
if args.prefetch_to_gpu:
train_hash = set(
[r.tobytes() for r in train_set.dataset.cpu().numpy()])
else:
train_hash = set([r.tobytes() for r in train_set.dataset])
all_hash = train_hash.copy()
all_hash.update(set([r.tobytes() for r in test_set.dataset]))
''' Comet Logging '''
experiment = Experiment(api_key=args.api_key,
disabled=not args.do_log,
project_name=args.project_name,
workspace=args.workspace)
experiment.set_name(args.namestr)
if not args.use_gcmc:
# modelD = TransD(args.num_ent, args.num_rel, args.embed_dim,\
# args.p).to(args.device)
modelD = TransE(args.num_ent, args.num_rel, args.embed_dim,\
args.p).to(args.device)
else:
decoder = SharedBilinearDecoder(args.num_rel, 2,
args.embed_dim).to(args.device)
modelD = SimpleGCMC(decoder, args.embed_dim, args.num_ent,
args.p).to(args.device)
''' Initialize Everything to None '''
fairD_gender, fairD_occupation, fairD_age, fairD_random = None, None, None, None
optimizer_fairD_gender, optimizer_fairD_occupation, \
optimizer_fairD_age, optimizer_fairD_random = None,None,None,None
gender_filter, occupation_filter, age_filter = None, None, None
if args.use_attr:
attr_data = [args.users, args.movies]
''' Initialize Discriminators '''
fairD_gender = GenderDiscriminator(args.use_1M,args.embed_dim,attr_data,\
'gender',use_cross_entropy=args.use_cross_entropy).to(args.device)
fairD_occupation = OccupationDiscriminator(args.use_1M,args.embed_dim,attr_data,\
attribute='occupation',use_cross_entropy=args.use_cross_entropy)
fairD_age = AgeDiscriminator(args.use_1M,args.embed_dim,attr_data,\
attribute='age',use_cross_entropy=args.use_cross_entropy)
''' Initialize Optimizers '''
if args.sample_mask:
gender_filter = AttributeFilter(args.embed_dim,
attribute='gender').to(args.device)
occupation_filter = AttributeFilter(args.embed_dim,
attribute='occupation').to(
args.device)
age_filter = AttributeFilter(args.embed_dim,
attribute='age').to(args.device)
optimizer_fairD_gender = optimizer(fairD_gender.parameters(),
'adam', args.lr)
optimizer_fairD_occupation = optimizer(
fairD_occupation.parameters(), 'adam', args.lr)
optimizer_fairD_age = optimizer(fairD_age.parameters(), 'adam',
args.lr)
elif args.use_occ_attr:
attr_data = [args.users, args.movies]
fairD_occupation = OccupationDiscriminator(args.use_1M,args.embed_dim,attr_data,\
attribute='occupation',use_cross_entropy=args.use_cross_entropy)
optimizer_fairD_occupation = optimizer(fairD_occupation.parameters(),
'adam', args.lr)
elif args.use_gender_attr:
attr_data = [args.users, args.movies]
fairD_gender = GenderDiscriminator(args.use_1M,args.embed_dim,attr_data,\
'gender',use_cross_entropy=args.use_cross_entropy)
optimizer_fairD_gender = optimizer(fairD_gender.parameters(), 'adam',
args.lr)
elif args.use_age_attr:
attr_data = [args.users, args.movies]
fairD_age = AgeDiscriminator(args.use_1M,args.embed_dim,attr_data,\
attribute='age',use_cross_entropy=args.use_cross_entropy)
optimizer_fairD_age = optimizer(fairD_age.parameters(), 'adam',
args.lr)
elif args.use_random_attr:
attr_data = [args.users, args.movies]
fairD_random = RandomDiscriminator(args.use_1M,args.embed_dim,attr_data,\
'random',use_cross_entropy=args.use_cross_entropy).to(args.device)
# fairD_random = DemParDisc(args.use_1M,args.embed_dim,attr_data,\
# attribute='random',use_cross_entropy=args.use_cross_entropy)
optimizer_fairD_random = optimizer(fairD_random.parameters(), 'adam',
args.lr)
if args.load_transD:
modelD.load(args.saved_path)
if args.load_filters:
gender_filter.load(args.gender_filter_saved_path)
occupation_filter.load(args.occupation_filter_saved_path)
age_filter.load(args.age_filter_saved_path)
''' Create Sets '''
fairD_set = [fairD_gender, fairD_occupation, fairD_age, fairD_random]
filter_set = [gender_filter, occupation_filter, age_filter, None]
optimizer_fairD_set = [optimizer_fairD_gender, optimizer_fairD_occupation,\
optimizer_fairD_age,optimizer_fairD_random]
''' Initialize CUDA if Available '''
if args.use_cuda:
for fairD, filter_ in zip(fairD_set, filter_set):
if fairD is not None:
fairD.to(args.device)
if filter_ is not None:
filter_.to(args.device)
if args.use_gcmc:
if args.sample_mask and not args.use_trained_filters:
optimizerD = optimizer(list(modelD.parameters()) + \
list(gender_filter.parameters()) + \
list(occupation_filter.parameters()) + \
list(age_filter.parameters()), 'adam', args.lr)
# optimizer_fairD_gender = optimizer(list(fairD_gender.parameters()) + \
# list(gender_filter.parameters()),'adam', args.lr)
else:
optimizerD = optimizer(modelD.parameters(), 'adam', args.lr)
else:
optimizerD = optimizer(modelD.parameters(), 'adam_sparse', args.lr)
_cst_inds = torch.LongTensor(np.arange(args.num_ent, \
dtype=np.int64)[:,None]).to(args.device).repeat(1, args.batch_size//2)
_cst_s = torch.LongTensor(np.arange(args.batch_size // 2)).to(args.device)
_cst_s_nb = torch.LongTensor(
np.arange(args.batch_size // 2, args.batch_size)).to(args.device)
_cst_nb = torch.LongTensor(np.arange(args.batch_size)).to(args.device)
if args.prefetch_to_gpu:
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
num_workers=0,
collate_fn=collate_fn)
else:
train_loader = DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
num_workers=4,
pin_memory=True,
collate_fn=collate_fn)
if args.freeze_transD:
freeze_model(modelD)
if args.debug:
attr_data = [args.users, args.movies]
ipdb.set_trace()
''' Joint Training '''
if not args.dont_train:
with experiment.train():
for epoch in tqdm(range(1, args.num_epochs + 1)):
if epoch % args.valid_freq == 0 or epoch == 1:
with torch.no_grad():
if args.use_gcmc:
rmse, test_loss = test_gcmc(
test_set, args, modelD, filter_set)
else:
# l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set, args, all_hash,\
# modelD,subsample=20)
test_nce(test_set, args, modelD, epoch, experiment)
if args.use_attr:
test_gender(args, test_fairness_set, modelD,
fairD_gender, experiment, epoch,
filter_set)
test_occupation(args, test_fairness_set, modelD,
fairD_occupation, experiment, epoch,
filter_set)
test_age(args, test_fairness_set, modelD, fairD_age,
experiment, epoch, filter_set)
elif args.use_gender_attr:
test_gender(args, test_fairness_set, modelD,
fairD_gender, experiment, epoch,
filter_set)
elif args.use_occ_attr:
test_occupation(args, test_fairness_set, modelD,
fairD_occupation, experiment, epoch,
filter_set)
elif args.use_age_attr:
test_age(args, test_fairness_set, modelD, fairD_age,
experiment, epoch, filter_set)
elif args.use_random_attr:
test_random(args, test_fairness_set, modelD,
fairD_random, experiment, epoch,
filter_set)
# test_fairness(test_fairness_set,args,modelD,experiment,\
# fairD_random,attribute='random',\
# epoch=epoch)
if args.do_log: # Tensorboard logging
if args.use_gcmc:
experiment.log_metric("RMSE",
float(rmse),
step=epoch)
experiment.log_metric("Test Loss",
float(rmse),
step=epoch)
# else:
# experiment.log_metric("Mean Rank",float(avg_mr),step=epoch)
# experiment.log_metric("Mean Reciprocal Rank",\
# float(avg_mrr),step=epoch)
# experiment.log_metric("Hit @10",float(avg_h10),step=epoch)
# experiment.log_metric("Hit @5",float(avg_h5),step=epoch)
train(train_loader,epoch,args,train_hash,modelD,optimizerD,\
fairD_set,optimizer_fairD_set,filter_set,experiment)
gc.collect()
if epoch % (args.valid_freq * 5) == 0:
if args.use_gcmc:
rmse = test_gcmc(test_set, args, modelD)
else:
test_nce(test_set, args, modelD, epoch, experiment)
# l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set,args, all_hash,\
# modelD,subsample=20)
# if not args.use_gcmc:
# l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set,args, all_hash, modelD)
# joblib.dump({'l_ranks':l_ranks, 'r_ranks':r_ranks}, args.outname_base+'test_ranks.pkl', compress=9)
modelD.save(args.outname_base + 'D_final.pts')
if args.use_attr or args.use_gender_attr:
fairD_gender.save(args.outname_base + 'GenderFairD_final.pts')
if args.use_attr or args.use_occ_attr:
fairD_occupation.save(args.outname_base +
'OccupationFairD_final.pts')
if args.use_attr or args.use_age_attr:
fairD_age.save(args.outname_base + 'AgeFairD_final.pts')
if args.use_random_attr:
fairD_random.save(args.outname_base + 'RandomFairD_final.pts')
if args.sample_mask:
gender_filter.save(args.outname_base + 'GenderFilter.pts')
occupation_filter.save(args.outname_base + 'OccupationFilter.pts')
age_filter.save(args.outname_base + 'AgeFilter.pts')
constant = len(fairD_set) - fairD_set.count(None)
if args.test_new_disc:
if args.test_new_disc:
args.use_attr = True
''' Training Fresh Discriminators'''
args.freeze_transD = True
attr_data = [args.users, args.movies]
if args.use_random_attr:
new_fairD_random = DemParDisc(args.use_1M,args.embed_dim,attr_data,\
attribute='random',use_cross_entropy=args.use_cross_entropy).to(args.device)
new_optimizer_fairD_random = optimizer(
new_fairD_random.parameters(), 'adam', args.lr)
freeze_model(modelD)
with experiment.test():
''' Train Classifier '''
if args.use_gender_attr or args.use_attr:
train_gender(args,modelD,train_fairness_set,test_fairness_set,\
attr_data,experiment,filter_set)
if args.use_occ_attr or args.use_attr:
train_occupation(args,modelD,train_fairness_set,test_fairness_set,\
attr_data,experiment,filter_set)
if args.use_age_attr or args.use_attr:
train_age(args,modelD,train_fairness_set,test_fairness_set,\
attr_data,experiment,filter_set)
if args.use_random_attr:
train_random(args,modelD,train_fairness_set,test_fairness_set,\
attr_data,experiment,filter_set)
# train_fairness_classifier(train_fairness_set,args,modelD,experiment,new_fairD_random,\
# new_optimizer_fairD_random,epoch,filter_=None,retrain=False)
if args.report_bias:
gender_bias = calc_attribute_bias('Train',args,modelD,experiment,\
'gender',epoch,[gender_filter])
occ_bias = calc_attribute_bias('Train',args,modelD,experiment,\
'occupation',epoch,[occupation_filter])
age_bias = calc_attribute_bias('Train',args,modelD,experiment,\
'age',epoch,[age_filter])
gender_bias = calc_attribute_bias('Test',args,modelD,experiment,\
'gender',epoch,[gender_filter])
occ_bias = calc_attribute_bias('Test',args,modelD,experiment,\
'occupation',epoch,[occupation_filter])
age_bias = calc_attribute_bias('Test',args,modelD,experiment,\
'age',epoch,[age_filter])
experiment.end()
def train(args, use_comet : bool = True):
data_cls = funcs[args['dataset']]
model_cls = funcs[args['model']]
network = funcs[args['network']]
print ('[INFO] Getting dataset...')
data = data_cls()
data.load_data()
(x_train, y_train), (x_test, y_test) = (data.x_train, data.y_train), (data.x_test, data.y_test)
classes = data.mapping
# #Used for testing only
# x_train = x_train[:100, :, :]
# y_train = y_train[:100, :]
# x_test = x_test[:100, :, :]
# y_test = y_test[:100, :]
# print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
# print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
# #delete these lines
# distribute 90% test 10% val dataset with equal class distribution
(x_test, x_valid, y_test, y_valid) = train_test_split(x_test, y_test, test_size=0.2, random_state=42)
print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
print ('[INFO] Validation shape: ', x_valid.shape, y_valid.shape)
print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
print ('[INFO] Setting up the model..')
if args['network'] == 'lstmctc':
network_args = {'backbone' : args['backbone'],
'seq_model' : args['seq'],
'bi' : args['bi']
}
model = model_cls(network, data_cls, network_args)
else:
model = model_cls(network, data_cls)
print (model)
dataset = dict({
'x_train' : x_train,
'y_train' : y_train,
'x_valid' : x_valid,
'y_valid' : y_valid,
'x_test' : x_test,
'y_test' : y_test
})
if use_comet and args['find_lr'] == False:
#create an experiment with your api key
experiment = Experiment(api_key='WVBNRAfMLCBWslJAAsffxM4Gz',
project_name='iam_lines',
auto_param_logging=False)
print ('[INFO] Starting Training...')
#will log metrics with the prefix 'train_'
with experiment.train():
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
#will log metrics with the prefix 'test_'
with experiment.test():
score = model.evaluate(dataset, int(args['batch_size']))
print(f'[INFO] Test evaluation: {score*100}...')
metrics = {
'accuracy':score
}
experiment.log_metrics(metrics)
experiment.log_parameters(args)
experiment.log_dataset_hash(x_train) #creates and logs a hash of your data
experiment.end()
elif use_comet and args['find_lr'] == True:
_ = train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
FIND_LR=args['find_lr'],
name=args['network']
)
else :
print ('[INFO] Starting Training...')
train_model(
model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network']
)
print ('[INFO] Starting Testing...')
score = model.evaluate(dataset, args['batch_size'])
print(f'[INFO] Test evaluation: {score*100}...')
if args['weights']:
model.save_weights()
if args['save_model']:
model.save_model()
'max_steps': MAX_STEPS,
'use_predictors': USE_PREDICTORS,
'batch_size': BATCH_SIZE,
'hidden_sizes': str(HIDDEN_SIZES),
'step_size': STEP_SIZE,
'predictor_lr': PREDICTOR_LR,
'lr': LR,
'epochs': EPOCHS,
# 'max_grad_norm': MAX_GRAD_NORM,
'solver': 'MAX_GRAD_NORM',
'predictor_optimizer': 'SGD',
# 'exploration_prob': EXPLORATION_PROB,
'nonlinearity_after_predictor': True,
'fixed_zero_grad': True,
# 'predictor_hidden': PREDICTOR_HIDDEN,
'stopping_criterion': 'MaxGradNormSolver',
# 'predictor_n_opt_steps': PREDICTOR_N_OPT_STEPS
}
EXP = Experiment(project_name='EqProp', auto_metric_logging=False)
EXP.add_tag('linear predictor')
EXP.log_parameters(hparams)
comment = f'{MAX_STEPS}_steps'
if USE_PREDICTORS:
comment += '_predictors'
main()
EXP.end()
class CometMLMonitor(MonitorBase):
"""
Send scalar data and the graph to https://www.comet.ml.
Note:
1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack.
2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze.
Therefore the feature is disabled by default.
"""
def __init__(self, experiment=None, tags=None, **kwargs):
"""
Args:
experiment (comet_ml.Experiment): if provided, invalidate all other arguments
tags (list[str]): experiment tags
kwargs: arguments used to initialize :class:`comet_ml.Experiment`,
such as project name, API key, etc.
Refer to its documentation for details.
"""
if experiment is not None:
self._exp = experiment
assert tags is None and len(kwargs) == 0
else:
from comet_ml import Experiment
kwargs.setdefault(
'log_code', True
) # though it's not functioning, git patch logging requires it
kwargs.setdefault('auto_output_logging', None)
self._exp = Experiment(**kwargs)
if tags is not None:
self._exp.add_tags(tags)
self._exp.set_code("Code logging is impossible ...")
self._exp.log_dependency('tensorpack', __git_version__)
@property
def experiment(self):
"""
The :class:`comet_ml.Experiment` instance.
"""
return self._exp
def _before_train(self):
self._exp.set_model_graph(tf.get_default_graph())
@HIDE_DOC
def process_scalar(self, name, val):
self._exp.log_metric(name, val, step=self.global_step)
@HIDE_DOC
def process_image(self, name, val):
self._exp.set_step(self.global_step)
for idx, v in enumerate(val):
log_name = "{}_step{}{}".format(
name, self.global_step, "_" + str(idx) if len(val) > 1 else "")
self._exp.log_image(v,
image_format="jpeg",
name=log_name,
image_minmax=(0, 255))
def _after_train(self):
self._exp.end()
def _after_epoch(self):
self._exp.log_epoch_end(self.epoch_num)
class Logger:
def __init__(self, send_logs, tags, parameters, experiment=None):
self.stations = 5
self.send_logs = send_logs
if self.send_logs:
if experiment is None:
json_loc = glob.glob("./**/comet_token.json")[0]
with open(json_loc, "r") as f:
kwargs = json.load(f)
self.experiment = Experiment(**kwargs)
else:
self.experiment = experiment
self.sent_mb = 0
self.speed_window = deque(maxlen=100)
self.step_time = None
self.current_speed = 0
if self.send_logs:
if tags is not None:
self.experiment.add_tags(tags)
if parameters is not None:
self.experiment.log_parameters(parameters)
def begin_logging(self, episode_count, steps_per_ep, sigma, theta, step_time):
self.step_time = step_time
if self.send_logs:
self.experiment.log_parameter("Episode count", episode_count)
self.experiment.log_parameter("Steps per episode", steps_per_ep)
self.experiment.log_parameter("theta", theta)
self.experiment.log_parameter("sigma", sigma)
def log_round(self, states, reward, cumulative_reward, info, loss, observations, step):
self.experiment.log_histogram_3d(states, name="Observations", step=step)
info = [[j for j in i.split("|")] for i in info]
info = np.mean(np.array(info, dtype=np.float32), axis=0)
try:
# round_mb = np.mean([float(i.split("|")[0]) for i in info])
round_mb = info[0]
except Exception as e:
print(info)
print(reward)
raise e
self.speed_window.append(round_mb)
self.current_speed = np.mean(np.asarray(self.speed_window)/self.step_time)
self.sent_mb += round_mb
# CW = np.mean([float(i.split("|")[1]) for i in info])
CW = info[1]
# stations = np.mean([float(i.split("|")[2]) for i in info])
self.stations = info[2]
fairness = info[3]
if self.send_logs:
self.experiment.log_metric("Round reward", np.mean(reward), step=step)
self.experiment.log_metric("Per-ep reward", np.mean(cumulative_reward), step=step)
self.experiment.log_metric("Megabytes sent", self.sent_mb, step=step)
self.experiment.log_metric("Round megabytes sent", round_mb, step=step)
self.experiment.log_metric("Chosen CW", CW, step=step)
self.experiment.log_metric("Station count", self.stations, step=step)
self.experiment.log_metric("Current throughput", self.current_speed, step=step)
self.experiment.log_metric("Fairness index", fairness, step=step)
for i, obs in enumerate(observations):
self.experiment.log_metric(f"Observation {i}", obs, step=step)
self.experiment.log_metrics(loss, step=step)
def log_episode(self, cumulative_reward, speed, step):
if self.send_logs:
self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step)
self.experiment.log_metric("Speed", speed, step=step)
self.sent_mb = 0
self.last_speed = speed
self.speed_window = deque(maxlen=100)
self.current_speed = 0
def end(self):
if self.send_logs:
self.experiment.end()
def run_exp(self):
''' Run the experiment on given number of pilots. '''
# Pre-train
pretrain_weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, '_all')
if self.pretraining:
print('Pretraining...')
_ = self.pretrain(pretrain_weights_filename, self.patience)
for pilot_idx in range(1, self.n_pilots + 1):
if self.log_pilots:
experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='pilot{}'.format(pilot_idx), log_code=False, auto_param_logging=False)
else:
experiment = None
# Load pilot data
X_train, y_train, X_valid, y_valid, X_test, y_test = self.load_data(pilot_idx, valid_ratio=0.2)
# Construct model & load pre-trained weights if available
weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)
self.model = self.build_model(load_weights=True if self.pretraining else False,
weights_filename=pretrain_weights_filename)
# Train
print('First phase training - Pilot {}'.format(pilot_idx))
hist, best_epoch = self.train(X_train, y_train, X_valid, y_valid,
save_filename='{}{}.h5'.format(self.weights_savename_prefix, pilot_idx),
patience=self.patience, experiment=experiment)
self.histories.append(hist)
# Test (before extra training)
self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, False)
# Extra-train
if self.train_after_es:
hist_es = self.extra_train(pilot_idx, X_valid, y_valid, weights_filename, hist.history['loss'][best_epoch], self.max_after_es_epochs) # New
self.histories_es.append(hist_es)
# Test (after extra training)
self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, True)
if self.log_pilots:
experiment.log_metrics({'Test accuracy' : self.test_score['accuracy'][-1]})
# Get t-SNE from intermediary outputs
layer_idxs = self.tsne_layer_idxs
get_output_functions = [ K.function([self.model.layers[0].input], [self.model.layers[idx].output]) for idx in layer_idxs]
# Training dataset
layer_outputs = [ get_output([X_train])[0] for get_output in get_output_functions ]
[ experiment.log_figure(figure_name='tsne_raw_train{}_layer{}'.format(pilot_idx, layer_idxs[idx]),
figure=tsne_plot(layer_outputs[idx], y_train, 20, title="t-SNE - Pilot {} - Layer {} (train)".format(pilot_idx, layer_idxs[idx])))
for idx in range(len(layer_idxs)) ]
# Testing dataset
layer_outputs = [ get_output([X_test])[0] for get_output in get_output_functions ]
[ experiment.log_figure(figure_name='tsne_raw_test{}_layer{}'.format(pilot_idx, layer_idxs[idx]),
figure=tsne_plot(layer_outputs[idx], y_test, 20, title="t-SNE - Pilot {} - Layer {} (test)".format(pilot_idx, layer_idxs[idx])))
for idx in range(len(layer_idxs)) ]
plt.close('all')
experiment.end()
# Export logs to Comet.ml
if self.comet_log:
self.log()
return self.test_score
def victim(kwargs=None):
def comet_pull_poison(craftstep):
for attempt in range(5):
try:
bytefile = craftexpt.get_asset(assets[craftstep])
if localrank == 0: print('==> poisoninputs-{} pulled'.format(craftstep))
poisoninputs = pickle.loads(bytefile)
return poisoninputs[:args.npoison]
except:
print(f'WARNING: comet pull attempt for craftstep {craftstep} failed on attempt {attempt}')
sleep(5)
if kwargs is not None:
for key in kwargs: globals()[key] = kwargs[key]
for key in argsmod: setattr(args, key, argsmod[key])
craftexpt = api.get_experiment(cometconfig["workspace"], args.craftproj, args.craftkey)
assets = {asset['step']: asset['assetId'] for asset in craftexpt.get_asset_list() if 'poisoninputs-' in asset['fileName']}
print('==> begin victim train')
trial = 0
while args.ntrial is None or trial < args.ntrial:
for craftstep in args.craftsteps:
experiment = Experiment(project_name=args.victimproj, auto_param_logging=False, auto_metric_logging=False, parse_args=False)
experiment.log_parameters(vars(args))
experiment.set_name(f'{args.craftkey[:5]}-{experiment.get_key()[:5]}')
experiment.add_tag(args.tag)
# experiment.add_tag(args.Xtag)
experiment.log_parameters(dict(craftstep=craftstep, trial=trial))
experiment.log_other('crafturl', craftexpt.url)
experiment.log_other('command', 'python ' + ' '.join(sys.argv))
if localrank == 0: print_command_and_args(args); print('crafturl: ' + craftexpt.url)
if 'victim.py' in sys.argv[0]:
poisoninputs = comet_pull_poison(craftstep)
if poisoninputs is None: experiment.end(); print(f'skipping craftstep {craftstep}'); continue
if args.savepoisondataset: package_poisoned_dataset(poisoninputs, xtrain, ytrain, xtarget, ytarget, ytargetadv, xvalid, yvalid, args, craftstep); experiment.end(); continue
# meta.init_weights(sess, pretrain_weights) # what we had before
meta.global_initialize(args, sess)
meta.poisoninputs.load(poisoninputs, sess)
trainstep = 0
for epoch in range(args.nvictimepoch):
tic = time()
lrnrate = lr_schedule(args.lrnrate, epoch, args.warmupperiod, args.schedule)
# log hidden layer features
if args.logfeat and epoch == args.nvictimepoch - 1:
feats = []
for victimfeed in feeddict_generator(xtrain, ytrain, lrnrate, meta, args, victim=True):
hiddens = sess.run(meta.hiddens, victimfeed)
for i, hidden in enumerate(hiddens):
if len(feats) <= i: feats.append(defaultdict(list))
feat = np.reshape(hidden, [-1, np.prod(hidden.shape[1:])])
appendfeats(feats[i], feat, victimfeed, ybase, ytarget, args.batchsize)
for i, feats_layer in enumerate(feats): comet_log_asset(experiment, f'feats_layer{i}', feats_layer, step=epoch)
# log validation acc
if epoch in np.round((args.nvictimepoch - 1) * np.linspace(0, 1, args.nvalidpoints) ** 2):
resVs = [] # validation
for _, validfeed, _ in feeddict_generator(xvalid, yvalid, lrnrate, meta, args, valid=True):
resV = sess.run(meta.resultV, validfeed)
resVs.append(resV)
experiment.log_metrics(avg_n_dicts(resVs), step=trainstep)
# train one epoch
for victimfeed in feeddict_generator(xtrain, ytrain, lrnrate, meta, args, victim=True):
_, resL = sess.run([meta.trainop, meta.resultL,], victimfeed)
if not trainstep % 200: experiment.log_metrics(resL, step=trainstep)
trainstep += 1
experiment.log_metric('elapsed', time() - tic, step=trainstep)
if args.saveweights: comet_log_asset_weights_and_buffers(epoch, experiment, meta, sess)
if not epoch % 20 and localrank == 0:
print(' | '.join([f'{args.craftkey[:5]}-{args.tag} | trial-{trial} | craftstep-{craftstep} | epoch {epoch} | elapsed {round(time() - tic, 2)}'] +
[f'{key} {trunc_decimal(val)}' for key, val in resL.items() if 'class' not in key] +
[f'{key} {trunc_decimal(val)}' for key, val in resV.items() if 'class' not in key]))
experiment.end()
trial += 1
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', type=str, default='gcn')
parser.add_argument('--type', type=str, default='base')
parser.add_argument('--runs', type=int, default=10)
parser.add_argument('--epochs', type=int, default=200)
parser.add_argument('--lr', type=float, default=0.001)
parser.add_argument('--hidden_dim', type=int, default=300)
parser.add_argument('--num_layers', type=int, default=5)
args = parser.parse_args()
assert args.model in ['gcn', 'sage', 'gin']
assert args.type in [
'base', 'transfer', 'transfer-damaged', 'self-transfer',
'self-transfer-damaged'
]
assert args.runs >= 1
assert args.epochs >= 1
assert args.lr > 0
assert args.hidden_dim > 0
assert args.num_layers > 0
# ---------------------------------------------------
# MODEL
# ---------------------------------------------------
model = networks[args.model](in_channels=dataset.num_features,
hidden_channels=args.hidden_dim,
out_channels=dataset.num_tasks,
num_conv_layers=args.num_layers).to(device)
# ---------------------------------------------------
# EXPERIMENT DETAILS
# ---------------------------------------------------
print('Graph Classification Experiment')
print('Mol_HIV task')
print(exp_description[args.type])
print('----------------------------------------------')
print('Model: {}'.format(args.model))
print('Number of runs: {}'.format(args.runs))
print('Number of epochs: {}'.format(args.epochs))
print('Learning rate: {}'.format(args.lr))
print()
print(model)
# ---------------------------------------------------
# EXPERIMENT LOOP
# ---------------------------------------------------
for run in range(args.runs):
print()
print('Run #{}'.format(run + 1))
# Model initialisation
if args.type == 'base':
model.reset_parameters()
elif args.type in ['transfer', 'transfer-damaged']:
# Pretrain on Mol-BBBP
model.reset_parameters()
bbbp_optimiser = optim.Adam(model.parameters(), lr=0.001)
to_damage = args.type == 'transfer-damaged'
print('Pretraining model on Mol-BBBP...')
best_val_acc = pretrain_molbbbp(model,
device,
bbbp_evaluator,
bbbp_optimiser,
args.model,
damage=to_damage)
print('Validation accuracy: {:.3}'.format(best_val_acc))
model.load_state_dict(
torch.load('molbbbp_models/{}_molbbbp.pth'.format(args.model)))
elif args.type in ['self-transfer', 'self-transfer-damaged']:
# Pretrain on Mol-HIV Source Split
model.reset_parameters()
source_optimiser = optim.Adam(model.parameters(), lr=0.001)
to_damage = args.type == 'self-transfer-damaged'
print('Pretraining model on Mol-HIV Source Task...')
best_val_acc = pretrain_source_molhiv(model,
device,
evaluator,
source_optimiser,
args.model,
damage=to_damage)
print('Validation accuracy: {:.3}'.format(best_val_acc))
model.load_state_dict(
torch.load('molhiv/{}_source_molhiv.pth'.format(args.model)))
# Comet Experiment
experiment = Experiment(project_name='graph-classification',
display_summary_level=0,
auto_metric_logging=False)
experiment.add_tags([args.model, args.type])
experiment.log_parameters({
'hidden_dim': args.hidden_dim,
'num_features': dataset.num_features,
'num_classes': dataset.num_tasks,
'learning_rate': args.lr,
'num_epochs': args.epochs,
})
# Mol-HIV Target Training
print('Training on Mol-HIV')
optimizer = optim.Adam(model.parameters(), args.lr)
for epoch in tqdm(range(args.epochs)):
train_loss = train(model, device, target_loader, optimizer)
train_performance = eval(model, device, target_loader, evaluator)
experiment.log_metric('train_loss', train_loss.item(), step=epoch)
experiment.log_metric('train_roc-auc',
train_performance[dataset.eval_metric],
step=epoch)
experiment.end()
def comet_lgbm(save_path):
from comet_ml import Experiment
exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj",
project_name="baseline", workspace="gdreiman1")
exp.log_code = True
import pickle
import pandas as pd
import lightgbm as lgb
import numpy as np
import sklearn
import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_fscore_support as prf
#%%
def single_roc(y_preds,y_true):
from sklearn.metrics import roc_curve, auc,precision_recall_curve
fpr, tpr, _ = roc_curve(y_true, y_preds)
roc_auc = auc(fpr, tpr)
plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
precision, recall, thresholds = precision_recall_curve(y_true, y_preds)
plt.plot(recall, precision, color='blue',
lw=lw, label='Precision vs Recall')
# show the plot
plt.legend(loc="lower right")
plt.show()
def multi_roc(y_preds,y_true,name,n_classes):
import collections
nested_dict = lambda: collections.defaultdict(nested_dict)
data_store = nested_dict()
from sklearn.metrics import roc_curve, auc
from scipy import interp
from itertools import cycle
lw = 2
name_store = ['Active', 'Inactive', 'Inconclusive']
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_preds[:, i])
roc_auc[i] = auc(fpr[i], tpr[i])
# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(y_true[:, i].ravel(), y_preds[:, i].ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
# Compute macro-average ROC curve and ROC area
# First aggregate all false positive rates
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))
# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
mean_tpr += interp(all_fpr, fpr[i], tpr[i])
# Finally average it and compute AUC
mean_tpr /= n_classes
fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])
# Plot all ROC curves
plt.figure()
plt.plot(fpr["micro"], tpr["micro"],
label='micro-average ROC curve (area = {0:0.2f})'
''.format(roc_auc["micro"]),
color='deeppink', linestyle=':', linewidth=4)
plt.plot(fpr["macro"], tpr["macro"],
label='macro-average ROC curve (area = {0:0.2f})'
''.format(roc_auc["macro"]),
color='navy', linestyle=':', linewidth=4)
colors = cycle(['aqua', 'darkorange', 'cornflowerblue','green'])
for i, color in zip(range(n_classes), colors):
plt.plot(fpr[i], tpr[i], color=color, lw=lw,
label='ROC curve of '+ name_store[i]+'(area = {1:0.2f})'
''.format(i, roc_auc[i]))
plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
#plt.title('Multi-class ROC for '+name+' Split= '+str(count+1))
plt.title('Multi-class ROC for '+name)
plt.legend(loc="lower right")
#plt.show()
#%%
#save_path = r'C:\Users\gdrei\Dropbox\UCL\Thesis\May_13\AID_1345083_processed.pkl'
model_type = 'lgbm'
#get data cleaned
pickle_off = open(save_path,'rb')
activity_table=pickle.load(pickle_off)
pickle_off.close()
#get length of MFP
fp_length = len(activity_table.iloc[5]['MFP'])
from sklearn.preprocessing import StandardScaler, LabelEncoder
scaler = StandardScaler(copy = False)
le = LabelEncoder()
labels = le.fit_transform(activity_table['PUBCHEM_ACTIVITY_OUTCOME'])
#split data:
from sklearn.model_selection import StratifiedShuffleSplit
splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.5, train_size=None, random_state=2562)
X_mfp = np.concatenate(np.array(activity_table['MFP'])).ravel()
X_mfp = X_mfp.reshape((-1,fp_length))
for train_ind, test_ind in splitter.split(X_mfp,labels):
# standardize data
X_train_molchars_std = scaler.fit_transform(np.array(activity_table.iloc[train_ind,4:]))
X_test_molchars_std = scaler.transform(np.array(activity_table.iloc[test_ind,4:]))
X_train = np.concatenate((X_mfp[train_ind,:],X_train_molchars_std),axis = 1)
X_test = np.concatenate((X_mfp[test_ind,:],X_test_molchars_std),axis = 1)
y_train = labels[train_ind]
y_test = labels[test_ind]
#X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X,labels,test_size = .5, shuffle = True, stratify = labels, random_state = 2562)
bin_y_train, bin_y_test = [1 if x ==2 else x for x in y_train],[1 if x ==2 else x for x in y_test]
#do light gbm
#need to make a lib svm file
train_data = lgb.Dataset(X_train,label=y_train)
test_data = lgb.Dataset(X_test,label=y_test)
#make model class
lgbm_model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=500, subsample_for_bin=200000,
objective='binary', is_unbalance=True, min_split_gain=0.0, min_child_weight=0.001, min_child_samples=20, subsample=1.0,
subsample_freq=0, colsample_bytree=1.0, reg_alpha=0.0, reg_lambda=0.0, random_state=None, n_jobs=-1, silent=True,
importance_type='split')
#train model
trained_mod = lgbm_model.fit(X_train,y_train)
#predict classes and class_probs
test_class_preds = lgbm_model.predict(X_test)
test_prob_preds = lgbm_model.predict_proba(X_test)
#calculate Class report
class_rep = sklearn.metrics.classification_report(y_test,test_class_preds)
print(class_rep)
if len(set(y_test)) == 2:
single_roc(test_prob_preds[:,1],y_test)
prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None)
else:
from tensorflow.keras.utils import to_categorical
multi_roc(test_prob_preds,to_categorical(y_test),'',3)
prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None)
#%%
'''Comet Saving Zone'''
#get AID number
import ntpath
#get base file name
folder,base = ntpath.split(save_path)
#split file name at second _ assumes file save in AID_xxx_endinfo.pkl
AID, _,end_info = base.rpartition('_')
#save data location, AID info, and version info
exp.log_dataset_info(name = AID, version = end_info, path = save_path)
#save model params
exp.log_parameters(trained_mod.get_params())
#save metrics report to comet
if len(f_1) == 2:
for i,name in enumerate(['Active','Inactive']):
exp.log_metric('f1 class '+name, f_1[i])
exp.log_metric('Recall class'+name,rec[i])
exp.log_metric('Precision class'+name, prec[i])
else:
for i,name in enumerate(['Active','Inconclusive','Inactive']):
exp.log_metric('f1 class '+str(i), f_1[i])
exp.log_metric('Recall class'+str(i),rec[i])
exp.log_metric('Precision class'+str(i), prec[i])
#exp.log_metric('f1 class '+str(i), f_1[i])
#exp.log_metric('Recall class'+str(i),rec[i])
#exp.log_metric('Precision class'+str(i), prec[i])
exp.log_other('Classification Report',class_rep)
#save model in data_folder with comet experiement number associated
exp_num = exp.get_key()
model_save = folder+'\\'+model_type+'_'+exp_num+'.pkl'
pickle_on = open(model_save,'wb')
pickle.dump(trained_mod,pickle_on)
pickle_on.close()
#log trained model location
exp.log_other('Trained Model Path',model_save)
#save some informatvie tags:
tags = [AID,end_info,model_type]
exp.add_tags(tags)
#save ROC curve
exp.log_figure(figure_name = 'ROC-Pres/Recall',figure=plt)
plt.show()
#tell comet that the experiement is over
exp.end()
def startExperiment(self):
experiment = Experiment(api_key=self.API_KEY,
project_name=self.comet_project_name,
workspace=self.WORKSPACE,
log_code=True)
# Disable cuda support
if not self.cuda_support:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
print(self.experiment_name)
features_temp = []
if self.feature_name.name == FeatureEnums.mfcc.name:
self.features = DataGrabber.getMfccs5Seconds_128_256_64()
self.model_with_maxpool = True
elif self.feature_name.name == FeatureEnums.env.name:
self.features = DataGrabber.getAmplitude10()
elif self.feature_name.name == FeatureEnums.zero_crossing.name:
self.features = DataGrabber.getZeroCrossingRate256_64()
elif self.feature_name.name == FeatureEnums.chroma_stft.name:
self.features = DataGrabber.getChromaStft128_256_64()
self.model_with_maxpool = True
elif self.feature_name.name == FeatureEnums.chroma_cqt.name:
self.features = DataGrabber.getChromaCqt5Sec128_7_64()
self.model_with_maxpool = True
elif self.feature_name.name == FeatureEnums.rms.name:
self.features = DataGrabber.getRMS()
dir_files = os.listdir(self.DATASET_DIRECTORY_PATH)
dir_files.sort(key=lambda f: int(re.sub('\D', '', f)))
para_count = 0
for file in dir_files:
if "xml" in file:
sample_number = int(file.split(".")[0][4:])
tree = ET.parse(
os.path.join(os.path.abspath(self.DATASET_DIRECTORY_PATH),
file))
root = tree.getroot()
class_labels = []
if self.vst_parameter is None:
for x in range(99):
class_labels.append(
int(root[x + 2].attrib["presetValue"]))
para_count = 99
else:
para_count = len(self.vst_parameter)
for para in self.vst_parameter:
class_labels.append(
int(root[para + 2].attrib["presetValue"]))
data = self.features[sample_number]
features_temp.append([data, class_labels])
del dir_files, self.features, tree, root
print("All Data Appended")
# Convert into a Panda dataframe
features_panda = pd.DataFrame(features_temp,
columns=['feature', 'class_label'])
del features_temp
# Convert features and corresponding classification labels into numpy arrays
feature_array = np.asarray(features_panda.feature.tolist())
value_array = np.asarray(features_panda.class_label.tolist())
del features_panda
le = LabelEncoder()
value_array = to_categorical(le.fit_transform(value_array))
# split the dataset
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(feature_array,
value_array,
test_size=0.2,
random_state=42)
if self.model_with_maxpool:
train_shape = (len(x_train[1]), len(x_train[1][1]))
model = self.createModel2D(train_shape, para_count)
else:
train_shape = len(x_train[1])
model = self.createModel1D(train_shape, para_count)
del feature_array, value_array
start = datetime.now()
# preparing callbacks
my_callbacks = [
# stops training after 20 epoch without improvement
tf.keras.callbacks.EarlyStopping(patience=20),
# saves model after each epoch
tf.keras.callbacks.ModelCheckpoint(
filepath=self.checkpoint_name + "_" + self.feature_name.name +
str(self.count_first_layer) + str(self.count_second_layer) +
str(self.count_third_layer) +
datetime.now().strftime("%m%d%Y%H%M%S") +
".{epoch:02d}-{val_loss:.4f}.h5"),
# reduce learning rate after 5 epochs without improvement
tf.keras.callbacks.ReduceLROnPlateau(monitor="loss",
factor=0.5,
patience=5,
min_lr=0.0001,
verbose=1)
]
print(model.summary())
print("start Training....")
history = model.fit(
x_train,
y_train,
batch_size=self.batch_size,
epochs=self.num_epochs,
validation_data=(x_test, y_test),
verbose=2,
callbacks=my_callbacks) # validation_data=(x_test, y_test)
duration = datetime.now() - start
print("Training completed in time: ", duration)
print("used feature: {}".format(self.feature_name))
experiment.end()
score = model.evaluate(x_train, y_train, verbose=0)
print("train accuracy: {}".format(score))
# experiment.log_metric("train_acc", score)
score = model.evaluate(x_test, y_test, verbose=0)
print('test accuracy: {}'.format(score))
class CometWriter(Writer):
'''
A Writer object to be used by all.experiments.Experiment.
Writes logs using comet.ml Requires an API key to be stored in .comet.config or as an environment variable.
Look at https://www.comet.ml/docs/python-sdk/advanced/#python-configuration for more info.
Args:
experiment (all.experiments.Experiment): The Experiment associated with the Writer object.
agent_name (str): The name of the Agent the Experiment is being performed on
env_name (str): The name of the environment the Experiment is being performed in
loss (bool, optional): Whether or not to log loss/scheduling metrics, or only evaluation and summary metrics.
logdir (str): The directory where run information is stored.
'''
def __init__(self, experiment, agent_name, env_name, loss=True, logdir='runs'):
self.env_name = env_name
self._experiment = experiment
self._loss = loss
try:
from comet_ml import Experiment
except ImportError as e:
print("Failed to import comet_ml. CometWriter requires that comet_ml be installed")
raise e
try:
self._comet = Experiment(project_name=env_name)
except ImportError as e:
print("See https://www.comet.ml/docs/python-sdk/warnings-errors/ for more info on this error.")
raise e
except ValueError as e:
print("See https://www.comet.ml/docs/python-sdk/advanced/#python-configuration for more info on this error.")
raise e
self._comet.set_name(agent_name)
self.log_dir = logdir
def add_loss(self, name, value, step="frame"):
if self._loss:
self.add_evaluation("loss/" + name, value, step)
def add_evaluation(self, name, value, step="frame"):
self._comet.log_metric(name, value, self._get_step(step))
def add_schedule(self, name, value, step="frame"):
if self._loss:
self.add_scalar(name, value, step)
def add_summary(self, name, mean, std, step="frame"):
self.add_evaluation(name + "/mean", mean, step)
self.add_evaluation(name + "/std", std, step)
def add_scalar(self, name, value, step="frame"):
self._comet.log_metric(name, value, self._get_step(step))
def _get_step(self, _type):
if _type == "frame":
return self._experiment.frame
if _type == "episode":
return self._experiment.episode
return _type
def close(self):
self._comet.end()
def train(args, use_comet: bool = True):
data_cls = funcs[args['dataset']]
model_cls = funcs[args['model']]
network = funcs[args['network']]
print('[INFO] Getting dataset...')
data = data_cls()
(x_train, y_train), (x_test, y_test) = data.load_data()
classes = data.mapping
# #Used for testing only
# x_train = x_train[:100, :, :]
# y_train = y_train[:100, :]
# x_test = x_test[:100, :, :]
# y_test = y_test[:100, :]
# print ('[INFO] Training shape: ', x_train.shape, y_train.shape)
# print ('[INFO] Test shape: ', x_test.shape, y_test.shape)
# #delete these lines
y_test_labels = [
np.where(y_test[idx] == 1)[0][0] for idx in range(len(y_test))
]
# distribute 90% test 10% val dataset with equal class distribution
(x_test, x_valid, y_test,
y_valid) = train_test_split(x_test,
y_test,
test_size=0.1,
stratify=y_test_labels,
random_state=42)
print('[INFO] Training shape: ', x_train.shape, y_train.shape)
print('[INFO] Validation shape: ', x_valid.shape, y_valid.shape)
print('[INFO] Test shape: ', x_test.shape, y_test.shape)
print('[INFO] Setting up the model..')
model = model_cls(network, data_cls)
print(model)
dataset = dict({
'x_train': x_train,
'y_train': y_train,
'x_valid': x_valid,
'y_valid': y_valid,
'x_test': x_test,
'y_test': y_test
})
if use_comet and args['find_lr'] == False:
#create an experiment with your api key
experiment = Experiment(api_key='INSERT API KEY',
project_name='emnist',
auto_param_logging=False)
print('[INFO] Starting Training...')
#will log metrics with the prefix 'train_'
with experiment.train():
_ = train_model(model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network'])
print('[INFO] Starting Testing...')
#will log metrics with the prefix 'test_'
with experiment.test():
loss, score = model.evaluate(dataset, args['batch_size'])
print(f'[INFO] Test evaluation: {score*100}')
metrics = {'loss': loss, 'accuracy': score}
experiment.log_metrics(metrics)
experiment.log_parameters(args)
experiment.log_dataset_hash(
x_train) #creates and logs a hash of your data
experiment.end()
elif use_comet and args['find_lr'] == True:
_ = train_model(model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
FIND_LR=args['find_lr'],
name=args['network'])
else:
print('[INFO] Starting Training...')
train_model(model,
dataset,
batch_size=args['batch_size'],
epochs=args['epochs'],
name=args['network'])
print('[INFO] Starting Testing...')
loss, score = model.evaluate(dataset, args['batch_size'])
print(f'[INFO] Test evaluation: {score*100}')
if args['weights']:
model.save_weights()
if args['save_model']:
model.save_model()
## Experiment 1: everything as normal, using .train():
experiment = Experiment()
experiment.add_tag("metrics")
results = {}
gbm = xgb.train(
params,
dtrain,
num_trees,
evals=watchlist,
early_stopping_rounds=50,
feval=rmspe_xg,
verbose_eval=True,
evals_result=results,
)
experiment.end()
## Experiment 2: no results (thus no metrics), using .train():
experiment = Experiment()
experiment.add_tag("no metrics")
gbm = xgb.train(
params,
dtrain,
num_trees,
evals=watchlist,
early_stopping_rounds=50,
feval=rmspe_xg,
verbose_eval=True,
)
experiment.end()
def cli_main():
parser = options.get_training_parser()
parser.add_argument(
"--comet-logging",
action="store_true",
help="Whether to use Comet.ML for logging",
)
args = options.parse_args_and_arch(parser)
logging = getattr(args, "comet_logging", False)
config = None
if logging:
PROJECT = "machine-translation"
if not keyring.get_password("comet", PROJECT):
comet_ml_api_key = getpass("Please enter the comet.ml API key: ")
keyring.set_password("comet", PROJECT, comet_ml_api_key)
else:
comet_ml_api_key = keyring.get_password("comet", PROJECT)
experiment = Experiment(
api_key=comet_ml_api_key,
project_name="machine-translation",
workspace="machine-translation",
auto_output_logging=None,
)
config = {
"api_key": comet_ml_api_key,
"experiment_key": experiment.get_key()
}
print("Proceeding with Comet.ML logging...")
if args.distributed_init_method is None:
distributed_utils.infer_init_method(args)
if args.distributed_init_method is not None:
# distributed training
if torch.cuda.device_count() > 1 and not args.distributed_no_spawn:
start_rank = args.distributed_rank
args.distributed_rank = None # assign automatically
torch.multiprocessing.spawn(
fn=distributed_main,
args=(args, config, start_rank),
nprocs=torch.cuda.device_count(),
)
else:
distributed_main(args.device_id, args, config)
elif args.distributed_world_size > 1:
# fallback for single node with multiple GPUs
assert args.distributed_world_size <= torch.cuda.device_count()
port = random.randint(10000, 20000)
args.distributed_init_method = "tcp://localhost:{port}".format(
port=port)
args.distributed_rank = None # set based on device id
if max(args.update_freq) > 1 and args.ddp_backend != "no_c10d":
print(
"| NOTE: you may get better performance with: --ddp-backend=no_c10d"
)
torch.multiprocessing.spawn(fn=distributed_main,
args=(args, config),
nprocs=args.distributed_world_size)
else:
# single GPU training
main(args, config=config)
if config:
experiment.end()
return start_indexs
def getNextIterInds(firstPicksList, fplist, bottom_to_select):
diverse_picks = mmp.LazyBitVectorPick(
fplist, len(fplist),
len(firstPicksList) + bottom_to_select, firstPicksList)
start_indexs = np.array(diverse_picks)
return start_indexs
with parallel_backend('multiprocessing'):
if selection_type == 'Diverse':
start_ind_list = Parallel(n_jobs=5)(
delayed(getNextIterInds)(
firstPicksList=i, fplist=j, bottom_to_select=k)
for i, j, k in zip(start_ind_list, fp_metalist,
diverse_size_list))
elif selection_type == 'Random':
start_ind_list = Parallel(n_jobs=5)(
delayed(getRandomIterInds)(
firstPicksList=i, fplist=j, bottom_to_select=k)
for i, j, k in zip(start_ind_list, fp_metalist,
diverse_size_list))
iter_num += 1
metrics_df = pd.DataFrame(metric_dict_list)
multi_dump_path = os.path.join('/home/gabriel/Dropbox/UCL/Thesis/Data/',
'second_diverse_GCNN_50epoch_iter_run.pkl')
exp.log_other('Metrics Dict Path', multi_dump_path)
metrics_df.to_pickle(multi_dump_path)
exp.end()
def train(self, train_data, cometml_key=None):
if cometml_key is not None:
experiment = Experiment(api_key=cometml_key,
project_name="dsgym-tgan",
workspace="baukebrenninkmeijer")
experiment.log_parameter('batch_size', self.batch_size)
experiment.log_parameter('embeddingDim', self.embeddingDim)
experiment.log_parameter('genDim', self.genDim)
experiment.log_parameter('disDim', self.disDim)
experiment.log_parameter('GAN version', 'TGAN')
# writer = SummaryWriter()
# train_data = monkey_with_train_data(train_data)
print('Transforming data...')
self.transformer = BGMTransformer(self.meta)
self.transformer.fit(train_data)
pickle.dump(self.transformer,
open(f'{self.working_dir}/transformer.pkl', 'wb'))
train_data = self.transformer.transform(train_data)
# ncp1 = sum(self.transformer.components[0])
# ncp2 = sum(self.transformer.components[1])
# for i in range(ncp1):
# for j in range(ncp2):
# cond1 = train_data[:, 1 + i] > 0
# cond2 = train_data[:, 2 + ncp1 + j]
# cond = np.logical_and(cond1, cond2)
#
# mean1 = train_data[cond, 0].mean()
# mean2 = train_data[cond, 1 + ncp1].mean()
#
# std1 = train_data[cond, 0].std()
# std2 = train_data[cond, 1 + ncp1].std()
# print(i, j, np.sum(cond), mean1, std1, mean2, std2, sep='\t')
# dataset = torch.utils.data.TensorDataset(torch.from_numpy(train_data.astype('float32')).to(self.device))
# loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, drop_last=True)
data_sampler = Sampler(train_data, self.transformer.output_info)
data_dim = self.transformer.output_dim
self.cond_generator = Cond(train_data, self.transformer.output_info)
self.generator = Generator(
self.embeddingDim + self.cond_generator.n_opt, self.genDim,
data_dim).to(self.device)
self.discriminator = Discriminator(
data_dim + self.cond_generator.n_opt, self.disDim).to(self.device)
optimizerG = optim.Adam(self.generator.parameters(),
lr=2e-4,
betas=(0.5, 0.9),
weight_decay=self.l2scale)
optimizerD = optim.Adam(self.discriminator.parameters(),
lr=2e-4,
betas=(0.5,
0.9)) #, weight_decay=self.l2scale)
# pickle.dump(self, open(f'{self.working_dir}/tgan_synthesizer.pkl', 'wb'))
# writer.add_graph(self.generator)
max_epoch = max(self.store_epoch)
assert self.batch_size % 2 == 0
mean = torch.zeros(self.batch_size,
self.embeddingDim,
device=self.device)
std = mean + 1
print('Starting training loop...')
steps_per_epoch = len(train_data) // self.batch_size
for i in tqdm(range(max_epoch)):
for id_ in tqdm(range(steps_per_epoch), leave=False):
fakez = torch.normal(mean=mean, std=std)
condvec = self.cond_generator.generate(self.batch_size)
if condvec is None:
c1, m1, col, opt = None, None, None, None
real = data_sampler.sample(self.batch_size, col, opt)
else:
c1, m1, col, opt = condvec
c1 = torch.from_numpy(c1).to(self.device)
m1 = torch.from_numpy(m1).to(self.device)
fakez = torch.cat([fakez, c1], dim=1)
perm = np.arange(self.batch_size)
np.random.shuffle(perm)
real = data_sampler.sample(self.batch_size, col[perm],
opt[perm])
c2 = c1[perm]
fake = self.generator(fakez)
fakeact = apply_activate(fake, self.transformer.output_info)
real = torch.from_numpy(real.astype('float32')).to(self.device)
if c1 is not None:
fake_cat = torch.cat([fakeact, c1], dim=1)
real_cat = torch.cat([real, c2], dim=1)
else:
real_cat = real
fake_cat = fake
# print(real_cat[0])
# print(fake_cat[0])
# assert 0
y_fake = self.discriminator(fake_cat)
y_real = self.discriminator(real_cat)
# loss_d = -(torch.log(torch.sigmoid(y_real) + 1e-4).mean()) - (torch.log(1. - torch.sigmoid(y_fake) + 1e-4).mean())
loss_d = -(torch.mean(y_real) - torch.mean(y_fake))
pen = calc_gradient_penalty(self.discriminator, real_cat,
fake_cat, self.device)
optimizerD.zero_grad()
pen.backward(retain_graph=True)
loss_d.backward()
optimizerD.step()
# for p in discriminator.parameters():
# p.data.clamp_(-0.05, 0.05)
fakez = torch.normal(mean=mean, std=std)
condvec = self.cond_generator.generate(self.batch_size)
if condvec is None:
c1, m1, col, opt = None, None, None, None
else:
c1, m1, col, opt = condvec
c1 = torch.from_numpy(c1).to(self.device)
m1 = torch.from_numpy(m1).to(self.device)
fakez = torch.cat([fakez, c1], dim=1)
fake = self.generator(fakez)
fakeact = apply_activate(fake, self.transformer.output_info)
if c1 is not None:
y_fake = self.discriminator(torch.cat([fakeact, c1],
dim=1))
else:
y_fake = self.discriminator(fakeact)
if condvec is None:
cross_entropy = 0
else:
cross_entropy = cond_loss(fake,
self.transformer.output_info, c1,
m1)
# loss_g = -torch.log(torch.sigmoid(y_fake) + 1e-4).mean() + cross_entropy
loss_g = -torch.mean(y_fake) + cross_entropy
optimizerG.zero_grad()
loss_g.backward()
optimizerG.step()
if cometml_key:
experiment.log_metric('Discriminator Loss', loss_d)
experiment.log_metric('Generator Loss', loss_g)
# print("---")
# print(fakeact[:, 0].mean(), fakeact[:, 0].std())
# print(fakeact[:, 1 + ncp1].mean(), fakeact[:, 1 + ncp1].std())
print(i + 1, loss_d.data, pen.data, loss_g.data, cross_entropy)
if cometml_key:
experiment.log_epoch_end(i)
if i + 1 in self.store_epoch:
print('Saving model')
torch.save(
{
"generator": self.generator.state_dict(),
"discriminator": self.discriminator.state_dict(),
}, "{}/model_{}.tar".format(self.working_dir, i + 1))
if cometml_key is not None:
experiment.end()
