def startExperiment(self): experiment = Experiment(api_key=self.API_KEY, project_name=self.comet_project_name, workspace=self.WORKSPACE, log_code=True) # Disable cuda support if not self.cuda_support: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' print(self.experiment_name) features_temp = [] data, class_labels = DataGrabber.getAudio5Sec() if self.vst_parameter is not None: for i, label in enumerate(class_labels): new_paras = [] for para in self.vst_parameter: new_paras.append(label[para]) class_labels[i] = new_paras features_temp.append([data, class_labels]) model = self.prepare_data_and_train(features_temp) del features_temp experiment.end()
def train(self): os.mkdir(self.paths['path']) if self.use_comet and self.api_key and self.project_name and self.workspace: experiment = Experiment(api_key=self.api_key, project_name=self.project_name, workspace=self.workspace) experiment.log_dataset_hash(self.train_dataset) experiment.add_tags([ str(self.architecture), "text_generation", f"nb_labels_{self.number_labels}" ]) with experiment.train(): hist = self.fit_dataset(self.train_dataset, self.val_dataset, self.epochs) experiment.end() elif self.use_comet: raise Exception( "Please provide an api_key, project_name and workspace for comet_ml" ) else: callbacks = self.callback_func( tensorboard_dir=self.paths['tensorboard_path'], checkpoint_path=self.paths['checkpoint_path']) hist = self.model.fit_dataset(self.train_dataset, self.val_dataset, self.epochs, callbacks) self.metrics = get_metrics(hist, "sparse_categorical_accuracy") self.export_weights(self.model) self.export_info(self.model_info) self.export_metrics(self.metrics) self.export_tokenizer(self.tokenizer) if self.do_zip_model: self.zip_model()
def pretrain(self, save_filename, patience): ''' Train & validate the model on full training dataset (all pilots) from scratch. Inputs: - save_filename: (string: 'weights_save_filename.h5'). - patience: Number of epochs without improvement before training stops (int). Output: - hist: Contains loss, acc, val_loss, val_acc metrics over epochs. (dictionnary) ''' X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], [] if self.log_pilots: experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='All pilots', log_code=False, auto_param_logging=False) for pilot_idx in range(1, self.n_pilots + 1): X_train_pilot, y_train_pilot, X_valid_pilot, y_valid_pilot, X_test_pilot, y_test_pilot = self.load_data(pilot_idx, valid_ratio=0.2) # Stacking training/validation datasets of each pilot into a big dataset if len(X_train)==0 and len(X_valid)==0: X_train = X_train_pilot y_train = y_train_pilot X_valid = X_valid_pilot y_valid = y_valid_pilot X_test = X_test_pilot y_test = y_test_pilot else: X_train = np.concatenate([X_train, X_train_pilot], axis=0) y_train = np.concatenate([y_train, y_train_pilot], axis=0) X_valid = np.concatenate([X_valid, X_valid_pilot], axis=0) y_valid = np.concatenate([y_valid, y_valid_pilot], axis=0) X_test = np.concatenate([X_test, X_test_pilot], axis=0) y_test = np.concatenate([y_test, y_test_pilot], axis=0) # Shuffle X_train, y_train = shuffle(X_train, y_train, random_state=0) X_valid, y_valid = shuffle(X_valid, y_valid, random_state=0) X_test, y_test = shuffle(X_test, y_test, random_state=0) # Build model self.model = self.build_model(load_weights=False) # Train on all pilots hist, _ = self.train(X_train, y_train, X_valid, y_valid, save_filename, patience) # Get some metrics score = self.test('all', save_filename, X_train, y_train, X_test, y_test, False) if self.log_pilots: experiment.log_metrics({'Test accuracy' : score}) experiment.end() return hist
class CometMLMonitor(MonitorBase): """ Send data to https://www.comet.ml. Note: 1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack. 2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze. Therefore the feature is disabled by default. """ def __init__(self, experiment=None, api_key=None, tags=None, **kwargs): """ Args: experiment (comet_ml.Experiment): if provided, invalidate all other arguments api_key (str): your comet.ml API key tags (list[str]): experiment tags kwargs: other arguments passed to :class:`comet_ml.Experiment`. """ if experiment is not None: self._exp = experiment assert api_key is None and tags is None and len(kwargs) == 0 else: from comet_ml import Experiment kwargs.setdefault( 'log_code', True ) # though it's not functioning, git patch logging requires it kwargs.setdefault('auto_output_logging', None) self._exp = Experiment(api_key=api_key, **kwargs) if tags is not None: self._exp.add_tags(tags) self._exp.set_code( "Code logging is impossible because there are too many files ...") self._exp.log_dependency('tensorpack', __git_version__) @property def experiment(self): """ The :class:`comet_ml.Experiment` instance. """ return self._exp def _before_train(self): self._exp.set_model_graph(tf.get_default_graph()) @HIDE_DOC def process_scalar(self, name, val): self._exp.log_metric(name, val, step=self.global_step) def _after_train(self): self._exp.end() def _after_epoch(self): self._exp.log_epoch_end(self.epoch_num)
def run(gpu_id, world_size, args): """ This is a single process that is linked to a single GPU :param gpu_id: The id of the GPU on the current node :param world_size: Total number of processes across nodes :param args: :return: """ torch.cuda.set_device(gpu_id) # The overall rank of this GPU process across multiple nodes global_process_rank = args.nr * args.gpus + gpu_id if global_process_rank == 0: experiment = Experiment(auto_output_logging="simple") else: experiment = Experiment(disabled=True) print( f"Running DDP model on Global Process with Rank: {global_process_rank }." ) setup(global_process_rank, world_size, args.backend) model = build_model() model.cuda(gpu_id) ddp_model = DDP(model, device_ids=[gpu_id]) criterion = nn.CrossEntropyLoss().cuda(gpu_id) optimizer = optim.Adam(model.parameters()) # Load training data train_dataset = torchvision.datasets.MNIST(root="./data", train=True, transform=transforms.ToTensor(), download=True) train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset, num_replicas=world_size, rank=global_process_rank) trainloader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0, pin_memory=True, sampler=train_sampler, ) for epoch in range(1, args.epochs + 1): train(ddp_model, optimizer, criterion, trainloader, epoch) cleanup() experiment.end()
def _train_with_comet(self, train_dataset, val_dataset): experiment = Experiment(api_key=self.api_key, project_name=self.project_name, workspace=self.workspace) experiment.log_dataset_hash(train_dataset) experiment.add_tags([ str(self.architecture), self.name, f"nb_labels_{self.label_encoder_classes_number}" ]) with experiment.train(): hist = self.fit_dataset(train_dataset, val_dataset) experiment.end() return hist
def startExperiment(self): experiment = Experiment(api_key=self.API_KEY, project_name=self.comet_project_name, workspace=self.WORKSPACE, log_code=True) # Disable cuda support if not self.cuda_support: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' print(self.experiment_name) features_temp = [] # for audio # data, class_labels = DataGrabber.getAudio5Sec() # if self.vst_parameter is not None: # for i, label in enumerate(class_labels): # new_paras = [] # for para in self.vst_parameter: # new_paras.append(label[para]) # class_labels[i] = new_paras self.features = DataGrabber.getZeroCrossingRate256_64() dir_files = os.listdir(self.DATASET_DIRECTORY_PATH) dir_files.sort(key=lambda f: int(re.sub('\D', '', f))) para_count = 0 for file in dir_files: if "xml" in file: sample_number = int(file.split(".")[0][4:]) tree = ET.parse(os.path.join(os.path.abspath(self.DATASET_DIRECTORY_PATH), file)) root = tree.getroot() class_labels = [] if self.vst_parameter is None: for x in range(99): class_labels.append(int(root[x + 2].attrib["presetValue"])) para_count = 99 else: para_count = len(self.vst_parameter) for para in self.vst_parameter: class_labels.append(int(root[para + 2].attrib["presetValue"])) data = self.features[sample_number] features_temp.append([data, class_labels]) del dir_files, self.features, tree, root print("All Data Appended") model = self.prepare_data_and_train(features_temp) del features_temp experiment.end()
class CometLogger(LightningLoggerBase): def __init__(self, *args, **kwargs): super(CometLogger, self).__init__() self.experiment = CometExperiment(*args, **kwargs) @rank_zero_only def log_hyperparams(self, params): self.experiment.log_parameters(vars(params)) @rank_zero_only def log_metrics(self, metrics, step_num): # self.experiment.set_epoch(self, metrics.get('epoch', 0)) self.experiment.log_metrics(metrics) @rank_zero_only def finalize(self, status): self.experiment.end()
def training_loop(x, y, cfg, exp_name="exp_1", device=torch.device('cuda'), starting_epoch=1): model = init_model(cfg, device) datasets = init_datasets(x, y, cfg) dataloaders = init_loaders(datasets, cfg) optim = init_optimizers(cfg, model) n_epochs = cfg.TRAIN.MAX_EPOCHS experiment = Experiment(project_name='mbm-pos-metrics', api_key='w4JbvdIWlas52xdwict9MwmyH') experiment.set_name(exp_name) hyper = { 'train_batch_size': cfg.TRAIN.BATCH_SIZE, 'age_lambda': cfg.MODEL.AGE_LAMBDA, 'n_epochs': cfg.TRAIN.MAX_EPOCHS, 'learning_rate': cfg.TRAIN.LEARNING_RATE, 'weight_decay': cfg.MODEL.WEIGHT_DECAY } experiment.log_parameters(hyper) print("Starting training loop!") for epoch in range(starting_epoch, n_epochs + 1): for phase in ['train', 'val', 'test']: print("Epoch {}/{} - {}".format(epoch, n_epochs, phase.upper())) forward_model(model, dataloaders[phase], optim, cfg, device, experiment, phase, current_epoch=epoch) # checkpoint model check = { 'model': model.state_dict(), 'optim': optim['optimizer'].state_dict() } torch.save( check, "../../storage/mbm/checkpoints/{}_{}.pth".format(exp_name, epoch)) experiment.end() return experiment
class Logger: def __init__(self, send_logs, tags, parameters): self.send_logs = send_logs if self.send_logs: self.experiment = Experiment(api_key="OZwyhJHyqzPZgHEpDFL1zxhyI", project_name="drilling-the-hole", workspace="wwydmanski") self.sent_mb = 0 if self.send_logs: if tags is not None: self.experiment.add_tags(tags) if parameters is not None: self.experiment.log_parameters(parameters) def begin_logging(self, episode_count, steps_per_ep): if self.send_logs: self.experiment.log_parameter("Episode count", episode_count) self.experiment.log_parameter("Max steps per episode", steps_per_ep) def log_round(self, actions, reward, cumulative_reward, angle, loss, step): if self.send_logs: self.experiment.log_metric("Round reward", reward, step=step) self.experiment.log_metric("Per-ep reward", cumulative_reward, step=step) self.experiment.log_metric("Action 1", actions[0], step=step) self.experiment.log_metric("Action 2", actions[1], step=step) self.experiment.log_metric("Current angle", angle, step=step) self.experiment.log_metrics(loss, step=step) def log_episode(self, cumulative_reward, state, step): if self.send_logs: self.experiment.log_metric("Angle", state[0], step=step) self.experiment.log_metric("Goal", state[1], step=step) self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step) def end(self): if self.send_logs: self.experiment.end()
def log_hyperparameters_to_comet(clf, experiment): for i in range(len(clf.cv_results_["params"])): exp = Experiment( workspace="s0lvang", project_name="ideal-pancake-hyperparameter", api_key=globals.flags.comet_api_key, ) exp.add_tag("hp_tuning") exp.add_tags(globals.comet_logger.get_tags()) for k, v in clf.cv_results_.items(): if k == "params": exp.log_parameters(v[i]) else: exp.log_metric(k, v[i]) exp.end() old_experiment = ExistingExperiment( api_key=globals.flags.comet_api_key, previous_experiment=experiment.get_key(), ) globals.comet_logger = old_experiment
def test_comet(self): """Test with a comet hook.""" from comet_ml import Experiment comet = Experiment(project_name="Testing", auto_output_logging="native") comet.log_dataset_info(name="Karcher", path="shonan") comet.add_tag("GaussNewton") comet.log_parameter("method", "GaussNewton") time = datetime.now() comet.set_name("GaussNewton-" + str(time.month) + "/" + str(time.day) + " " + str(time.hour) + ":" + str(time.minute) + ":" + str(time.second)) # I want to do some comet thing here def hook(optimizer, error): comet.log_metric("Karcher error", error, optimizer.iterations()) gtsam_optimize(self.optimizer, self.params, hook) comet.end() actual = self.optimizer.values() self.gtsamAssertEquals(actual.atRot3(KEY), self.expected)
class CometLogger(): def __init__(self): global comet_installed self._logging = False if comet_installed: try: self._experiment = Experiment(auto_metric_logging=False, display_summary_level=0) self._experiment.log_other("Created from", "sweetviz!") self._logging = True except: print( "ERROR: comet_ml is installed, but not configured properly (e.g. check API key setup). HTML reports will not be uploaded." ) def log_html(self, html_content): if self._logging: try: self._experiment.log_html(html_content) except: print( "comet_ml.log_html(): error occurred during call to log_html()." ) else: print( "comet_ml.log_html(): comet_ml is not installed or otherwise ready for logging." ) def end(self): if self._logging: try: self._experiment.end() except: print("comet_ml.end(): error occurred during call to end().") else: print( "comet_ml.end(): comet_ml is not installed or otherwise ready." )
def log(self, experiment=None): ''' Export all logs in the Comet.ml environment. See https://www.comet.ml/ for more details ''' # Initialize Comet.ml experience (naming, tags) for automatic logging project_name = 'Optimization' if self.comet_optimize else 'Summary' experiment_name = '{} - {} '.format(self.model_name, str(self.batch_size)) + ('ES+' if self.train_after_es else '') experiment_tags = [ self.model_name, self.monitor_val ] + (['ES+'] if self.train_after_es else []) + (['Pre-train'] if self.pretraining else []) if experiment == None: experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name=project_name, log_code=False, auto_param_logging=False, auto_metric_logging=False) experiment.set_name(experiment_name) experiment.add_tags(experiment_tags) # Export hyperparameters experiment.log_parameters(self.dataloader_params) experiment.log_parameters(self.training_params) # Export metrics values experiment.log_metrics({'Average accuracy' : np.mean(self.test_score['accuracy']), 'Std accuracy' : np.std(self.test_score['accuracy'])}) # Export metrics graphs for each pilot (accuracy, loss, confusion matrix) [ experiment.log_figure(figure_name='Confusion matrix {}'.format(pilot_idx), figure=plot_cm(self.conf_matrices, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] [ experiment.log_figure(figure_name='Loss pilot {}'.format(pilot_idx), figure=plot_loss(self.histories[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] fig, ax = plt.subplots(figsize=(10,6)) plot_full_barchart(self.test_score, n_pilots=self.n_pilots, title=' {} ConvNet model'.format(self.model_name), fig=fig) experiment.log_figure(figure_name='Accuracy barchart', figure=fig) if self.train_after_es: [ experiment.log_figure(figure_name='Loss pilot {} (ES+)'.format(pilot_idx), figure=plot_loss(self.histories_es[pilot_idx-1], pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] # Export model weights for each pilot [ experiment.log_asset('{}{}.h5'.format(self.weights_savename_prefix, pilot_idx)) for pilot_idx in range(1,self.n_pilots+1)] experiment.end()
def main(args): train_set = KBDataset(args.train_ratings, args.prefetch_to_gpu) test_set = KBDataset(args.test_ratings, args.prefetch_to_gpu) train_fairness_set = NodeClassification(args.users_train, args.prefetch_to_gpu) test_fairness_set = NodeClassification(args.users_test, args.prefetch_to_gpu) if args.prefetch_to_gpu: train_hash = set( [r.tobytes() for r in train_set.dataset.cpu().numpy()]) else: train_hash = set([r.tobytes() for r in train_set.dataset]) all_hash = train_hash.copy() all_hash.update(set([r.tobytes() for r in test_set.dataset])) ''' Comet Logging ''' experiment = Experiment(api_key=args.api_key, disabled=not args.do_log, project_name=args.project_name, workspace=args.workspace) experiment.set_name(args.namestr) if not args.use_gcmc: # modelD = TransD(args.num_ent, args.num_rel, args.embed_dim,\ # args.p).to(args.device) modelD = TransE(args.num_ent, args.num_rel, args.embed_dim,\ args.p).to(args.device) else: decoder = SharedBilinearDecoder(args.num_rel, 2, args.embed_dim).to(args.device) modelD = SimpleGCMC(decoder, args.embed_dim, args.num_ent, args.p).to(args.device) ''' Initialize Everything to None ''' fairD_gender, fairD_occupation, fairD_age, fairD_random = None, None, None, None optimizer_fairD_gender, optimizer_fairD_occupation, \ optimizer_fairD_age, optimizer_fairD_random = None,None,None,None gender_filter, occupation_filter, age_filter = None, None, None if args.use_attr: attr_data = [args.users, args.movies] ''' Initialize Discriminators ''' fairD_gender = GenderDiscriminator(args.use_1M,args.embed_dim,attr_data,\ 'gender',use_cross_entropy=args.use_cross_entropy).to(args.device) fairD_occupation = OccupationDiscriminator(args.use_1M,args.embed_dim,attr_data,\ attribute='occupation',use_cross_entropy=args.use_cross_entropy) fairD_age = AgeDiscriminator(args.use_1M,args.embed_dim,attr_data,\ attribute='age',use_cross_entropy=args.use_cross_entropy) ''' Initialize Optimizers ''' if args.sample_mask: gender_filter = AttributeFilter(args.embed_dim, attribute='gender').to(args.device) occupation_filter = AttributeFilter(args.embed_dim, attribute='occupation').to( args.device) age_filter = AttributeFilter(args.embed_dim, attribute='age').to(args.device) optimizer_fairD_gender = optimizer(fairD_gender.parameters(), 'adam', args.lr) optimizer_fairD_occupation = optimizer( fairD_occupation.parameters(), 'adam', args.lr) optimizer_fairD_age = optimizer(fairD_age.parameters(), 'adam', args.lr) elif args.use_occ_attr: attr_data = [args.users, args.movies] fairD_occupation = OccupationDiscriminator(args.use_1M,args.embed_dim,attr_data,\ attribute='occupation',use_cross_entropy=args.use_cross_entropy) optimizer_fairD_occupation = optimizer(fairD_occupation.parameters(), 'adam', args.lr) elif args.use_gender_attr: attr_data = [args.users, args.movies] fairD_gender = GenderDiscriminator(args.use_1M,args.embed_dim,attr_data,\ 'gender',use_cross_entropy=args.use_cross_entropy) optimizer_fairD_gender = optimizer(fairD_gender.parameters(), 'adam', args.lr) elif args.use_age_attr: attr_data = [args.users, args.movies] fairD_age = AgeDiscriminator(args.use_1M,args.embed_dim,attr_data,\ attribute='age',use_cross_entropy=args.use_cross_entropy) optimizer_fairD_age = optimizer(fairD_age.parameters(), 'adam', args.lr) elif args.use_random_attr: attr_data = [args.users, args.movies] fairD_random = RandomDiscriminator(args.use_1M,args.embed_dim,attr_data,\ 'random',use_cross_entropy=args.use_cross_entropy).to(args.device) # fairD_random = DemParDisc(args.use_1M,args.embed_dim,attr_data,\ # attribute='random',use_cross_entropy=args.use_cross_entropy) optimizer_fairD_random = optimizer(fairD_random.parameters(), 'adam', args.lr) if args.load_transD: modelD.load(args.saved_path) if args.load_filters: gender_filter.load(args.gender_filter_saved_path) occupation_filter.load(args.occupation_filter_saved_path) age_filter.load(args.age_filter_saved_path) ''' Create Sets ''' fairD_set = [fairD_gender, fairD_occupation, fairD_age, fairD_random] filter_set = [gender_filter, occupation_filter, age_filter, None] optimizer_fairD_set = [optimizer_fairD_gender, optimizer_fairD_occupation,\ optimizer_fairD_age,optimizer_fairD_random] ''' Initialize CUDA if Available ''' if args.use_cuda: for fairD, filter_ in zip(fairD_set, filter_set): if fairD is not None: fairD.to(args.device) if filter_ is not None: filter_.to(args.device) if args.use_gcmc: if args.sample_mask and not args.use_trained_filters: optimizerD = optimizer(list(modelD.parameters()) + \ list(gender_filter.parameters()) + \ list(occupation_filter.parameters()) + \ list(age_filter.parameters()), 'adam', args.lr) # optimizer_fairD_gender = optimizer(list(fairD_gender.parameters()) + \ # list(gender_filter.parameters()),'adam', args.lr) else: optimizerD = optimizer(modelD.parameters(), 'adam', args.lr) else: optimizerD = optimizer(modelD.parameters(), 'adam_sparse', args.lr) _cst_inds = torch.LongTensor(np.arange(args.num_ent, \ dtype=np.int64)[:,None]).to(args.device).repeat(1, args.batch_size//2) _cst_s = torch.LongTensor(np.arange(args.batch_size // 2)).to(args.device) _cst_s_nb = torch.LongTensor( np.arange(args.batch_size // 2, args.batch_size)).to(args.device) _cst_nb = torch.LongTensor(np.arange(args.batch_size)).to(args.device) if args.prefetch_to_gpu: train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=0, collate_fn=collate_fn) else: train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=4, pin_memory=True, collate_fn=collate_fn) if args.freeze_transD: freeze_model(modelD) if args.debug: attr_data = [args.users, args.movies] ipdb.set_trace() ''' Joint Training ''' if not args.dont_train: with experiment.train(): for epoch in tqdm(range(1, args.num_epochs + 1)): if epoch % args.valid_freq == 0 or epoch == 1: with torch.no_grad(): if args.use_gcmc: rmse, test_loss = test_gcmc( test_set, args, modelD, filter_set) else: # l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set, args, all_hash,\ # modelD,subsample=20) test_nce(test_set, args, modelD, epoch, experiment) if args.use_attr: test_gender(args, test_fairness_set, modelD, fairD_gender, experiment, epoch, filter_set) test_occupation(args, test_fairness_set, modelD, fairD_occupation, experiment, epoch, filter_set) test_age(args, test_fairness_set, modelD, fairD_age, experiment, epoch, filter_set) elif args.use_gender_attr: test_gender(args, test_fairness_set, modelD, fairD_gender, experiment, epoch, filter_set) elif args.use_occ_attr: test_occupation(args, test_fairness_set, modelD, fairD_occupation, experiment, epoch, filter_set) elif args.use_age_attr: test_age(args, test_fairness_set, modelD, fairD_age, experiment, epoch, filter_set) elif args.use_random_attr: test_random(args, test_fairness_set, modelD, fairD_random, experiment, epoch, filter_set) # test_fairness(test_fairness_set,args,modelD,experiment,\ # fairD_random,attribute='random',\ # epoch=epoch) if args.do_log: # Tensorboard logging if args.use_gcmc: experiment.log_metric("RMSE", float(rmse), step=epoch) experiment.log_metric("Test Loss", float(rmse), step=epoch) # else: # experiment.log_metric("Mean Rank",float(avg_mr),step=epoch) # experiment.log_metric("Mean Reciprocal Rank",\ # float(avg_mrr),step=epoch) # experiment.log_metric("Hit @10",float(avg_h10),step=epoch) # experiment.log_metric("Hit @5",float(avg_h5),step=epoch) train(train_loader,epoch,args,train_hash,modelD,optimizerD,\ fairD_set,optimizer_fairD_set,filter_set,experiment) gc.collect() if epoch % (args.valid_freq * 5) == 0: if args.use_gcmc: rmse = test_gcmc(test_set, args, modelD) else: test_nce(test_set, args, modelD, epoch, experiment) # l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set,args, all_hash,\ # modelD,subsample=20) # if not args.use_gcmc: # l_ranks,r_ranks,avg_mr,avg_mrr,avg_h10,avg_h5 = test(test_set,args, all_hash, modelD) # joblib.dump({'l_ranks':l_ranks, 'r_ranks':r_ranks}, args.outname_base+'test_ranks.pkl', compress=9) modelD.save(args.outname_base + 'D_final.pts') if args.use_attr or args.use_gender_attr: fairD_gender.save(args.outname_base + 'GenderFairD_final.pts') if args.use_attr or args.use_occ_attr: fairD_occupation.save(args.outname_base + 'OccupationFairD_final.pts') if args.use_attr or args.use_age_attr: fairD_age.save(args.outname_base + 'AgeFairD_final.pts') if args.use_random_attr: fairD_random.save(args.outname_base + 'RandomFairD_final.pts') if args.sample_mask: gender_filter.save(args.outname_base + 'GenderFilter.pts') occupation_filter.save(args.outname_base + 'OccupationFilter.pts') age_filter.save(args.outname_base + 'AgeFilter.pts') constant = len(fairD_set) - fairD_set.count(None) if args.test_new_disc: if args.test_new_disc: args.use_attr = True ''' Training Fresh Discriminators''' args.freeze_transD = True attr_data = [args.users, args.movies] if args.use_random_attr: new_fairD_random = DemParDisc(args.use_1M,args.embed_dim,attr_data,\ attribute='random',use_cross_entropy=args.use_cross_entropy).to(args.device) new_optimizer_fairD_random = optimizer( new_fairD_random.parameters(), 'adam', args.lr) freeze_model(modelD) with experiment.test(): ''' Train Classifier ''' if args.use_gender_attr or args.use_attr: train_gender(args,modelD,train_fairness_set,test_fairness_set,\ attr_data,experiment,filter_set) if args.use_occ_attr or args.use_attr: train_occupation(args,modelD,train_fairness_set,test_fairness_set,\ attr_data,experiment,filter_set) if args.use_age_attr or args.use_attr: train_age(args,modelD,train_fairness_set,test_fairness_set,\ attr_data,experiment,filter_set) if args.use_random_attr: train_random(args,modelD,train_fairness_set,test_fairness_set,\ attr_data,experiment,filter_set) # train_fairness_classifier(train_fairness_set,args,modelD,experiment,new_fairD_random,\ # new_optimizer_fairD_random,epoch,filter_=None,retrain=False) if args.report_bias: gender_bias = calc_attribute_bias('Train',args,modelD,experiment,\ 'gender',epoch,[gender_filter]) occ_bias = calc_attribute_bias('Train',args,modelD,experiment,\ 'occupation',epoch,[occupation_filter]) age_bias = calc_attribute_bias('Train',args,modelD,experiment,\ 'age',epoch,[age_filter]) gender_bias = calc_attribute_bias('Test',args,modelD,experiment,\ 'gender',epoch,[gender_filter]) occ_bias = calc_attribute_bias('Test',args,modelD,experiment,\ 'occupation',epoch,[occupation_filter]) age_bias = calc_attribute_bias('Test',args,modelD,experiment,\ 'age',epoch,[age_filter]) experiment.end()
def train(args, use_comet : bool = True): data_cls = funcs[args['dataset']] model_cls = funcs[args['model']] network = funcs[args['network']] print ('[INFO] Getting dataset...') data = data_cls() data.load_data() (x_train, y_train), (x_test, y_test) = (data.x_train, data.y_train), (data.x_test, data.y_test) classes = data.mapping # #Used for testing only # x_train = x_train[:100, :, :] # y_train = y_train[:100, :] # x_test = x_test[:100, :, :] # y_test = y_test[:100, :] # print ('[INFO] Training shape: ', x_train.shape, y_train.shape) # print ('[INFO] Test shape: ', x_test.shape, y_test.shape) # #delete these lines # distribute 90% test 10% val dataset with equal class distribution (x_test, x_valid, y_test, y_valid) = train_test_split(x_test, y_test, test_size=0.2, random_state=42) print ('[INFO] Training shape: ', x_train.shape, y_train.shape) print ('[INFO] Validation shape: ', x_valid.shape, y_valid.shape) print ('[INFO] Test shape: ', x_test.shape, y_test.shape) print ('[INFO] Setting up the model..') if args['network'] == 'lstmctc': network_args = {'backbone' : args['backbone'], 'seq_model' : args['seq'], 'bi' : args['bi'] } model = model_cls(network, data_cls, network_args) else: model = model_cls(network, data_cls) print (model) dataset = dict({ 'x_train' : x_train, 'y_train' : y_train, 'x_valid' : x_valid, 'y_valid' : y_valid, 'x_test' : x_test, 'y_test' : y_test }) if use_comet and args['find_lr'] == False: #create an experiment with your api key experiment = Experiment(api_key='WVBNRAfMLCBWslJAAsffxM4Gz', project_name='iam_lines', auto_param_logging=False) print ('[INFO] Starting Training...') #will log metrics with the prefix 'train_' with experiment.train(): _ = train_model( model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], name=args['network'] ) print ('[INFO] Starting Testing...') #will log metrics with the prefix 'test_' with experiment.test(): score = model.evaluate(dataset, int(args['batch_size'])) print(f'[INFO] Test evaluation: {score*100}...') metrics = { 'accuracy':score } experiment.log_metrics(metrics) experiment.log_parameters(args) experiment.log_dataset_hash(x_train) #creates and logs a hash of your data experiment.end() elif use_comet and args['find_lr'] == True: _ = train_model( model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], FIND_LR=args['find_lr'], name=args['network'] ) else : print ('[INFO] Starting Training...') train_model( model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], name=args['network'] ) print ('[INFO] Starting Testing...') score = model.evaluate(dataset, args['batch_size']) print(f'[INFO] Test evaluation: {score*100}...') if args['weights']: model.save_weights() if args['save_model']: model.save_model()
'max_steps': MAX_STEPS, 'use_predictors': USE_PREDICTORS, 'batch_size': BATCH_SIZE, 'hidden_sizes': str(HIDDEN_SIZES), 'step_size': STEP_SIZE, 'predictor_lr': PREDICTOR_LR, 'lr': LR, 'epochs': EPOCHS, # 'max_grad_norm': MAX_GRAD_NORM, 'solver': 'MAX_GRAD_NORM', 'predictor_optimizer': 'SGD', # 'exploration_prob': EXPLORATION_PROB, 'nonlinearity_after_predictor': True, 'fixed_zero_grad': True, # 'predictor_hidden': PREDICTOR_HIDDEN, 'stopping_criterion': 'MaxGradNormSolver', # 'predictor_n_opt_steps': PREDICTOR_N_OPT_STEPS } EXP = Experiment(project_name='EqProp', auto_metric_logging=False) EXP.add_tag('linear predictor') EXP.log_parameters(hparams) comment = f'{MAX_STEPS}_steps' if USE_PREDICTORS: comment += '_predictors' main() EXP.end()
class CometMLMonitor(MonitorBase): """ Send scalar data and the graph to https://www.comet.ml. Note: 1. comet_ml requires you to `import comet_ml` before importing tensorflow or tensorpack. 2. The "automatic output logging" feature of comet_ml will make the training progress bar appear to freeze. Therefore the feature is disabled by default. """ def __init__(self, experiment=None, tags=None, **kwargs): """ Args: experiment (comet_ml.Experiment): if provided, invalidate all other arguments tags (list[str]): experiment tags kwargs: arguments used to initialize :class:`comet_ml.Experiment`, such as project name, API key, etc. Refer to its documentation for details. """ if experiment is not None: self._exp = experiment assert tags is None and len(kwargs) == 0 else: from comet_ml import Experiment kwargs.setdefault( 'log_code', True ) # though it's not functioning, git patch logging requires it kwargs.setdefault('auto_output_logging', None) self._exp = Experiment(**kwargs) if tags is not None: self._exp.add_tags(tags) self._exp.set_code("Code logging is impossible ...") self._exp.log_dependency('tensorpack', __git_version__) @property def experiment(self): """ The :class:`comet_ml.Experiment` instance. """ return self._exp def _before_train(self): self._exp.set_model_graph(tf.get_default_graph()) @HIDE_DOC def process_scalar(self, name, val): self._exp.log_metric(name, val, step=self.global_step) @HIDE_DOC def process_image(self, name, val): self._exp.set_step(self.global_step) for idx, v in enumerate(val): log_name = "{}_step{}{}".format( name, self.global_step, "_" + str(idx) if len(val) > 1 else "") self._exp.log_image(v, image_format="jpeg", name=log_name, image_minmax=(0, 255)) def _after_train(self): self._exp.end() def _after_epoch(self): self._exp.log_epoch_end(self.epoch_num)
class Logger: def __init__(self, send_logs, tags, parameters, experiment=None): self.stations = 5 self.send_logs = send_logs if self.send_logs: if experiment is None: json_loc = glob.glob("./**/comet_token.json")[0] with open(json_loc, "r") as f: kwargs = json.load(f) self.experiment = Experiment(**kwargs) else: self.experiment = experiment self.sent_mb = 0 self.speed_window = deque(maxlen=100) self.step_time = None self.current_speed = 0 if self.send_logs: if tags is not None: self.experiment.add_tags(tags) if parameters is not None: self.experiment.log_parameters(parameters) def begin_logging(self, episode_count, steps_per_ep, sigma, theta, step_time): self.step_time = step_time if self.send_logs: self.experiment.log_parameter("Episode count", episode_count) self.experiment.log_parameter("Steps per episode", steps_per_ep) self.experiment.log_parameter("theta", theta) self.experiment.log_parameter("sigma", sigma) def log_round(self, states, reward, cumulative_reward, info, loss, observations, step): self.experiment.log_histogram_3d(states, name="Observations", step=step) info = [[j for j in i.split("|")] for i in info] info = np.mean(np.array(info, dtype=np.float32), axis=0) try: # round_mb = np.mean([float(i.split("|")[0]) for i in info]) round_mb = info[0] except Exception as e: print(info) print(reward) raise e self.speed_window.append(round_mb) self.current_speed = np.mean(np.asarray(self.speed_window)/self.step_time) self.sent_mb += round_mb # CW = np.mean([float(i.split("|")[1]) for i in info]) CW = info[1] # stations = np.mean([float(i.split("|")[2]) for i in info]) self.stations = info[2] fairness = info[3] if self.send_logs: self.experiment.log_metric("Round reward", np.mean(reward), step=step) self.experiment.log_metric("Per-ep reward", np.mean(cumulative_reward), step=step) self.experiment.log_metric("Megabytes sent", self.sent_mb, step=step) self.experiment.log_metric("Round megabytes sent", round_mb, step=step) self.experiment.log_metric("Chosen CW", CW, step=step) self.experiment.log_metric("Station count", self.stations, step=step) self.experiment.log_metric("Current throughput", self.current_speed, step=step) self.experiment.log_metric("Fairness index", fairness, step=step) for i, obs in enumerate(observations): self.experiment.log_metric(f"Observation {i}", obs, step=step) self.experiment.log_metrics(loss, step=step) def log_episode(self, cumulative_reward, speed, step): if self.send_logs: self.experiment.log_metric("Cumulative reward", cumulative_reward, step=step) self.experiment.log_metric("Speed", speed, step=step) self.sent_mb = 0 self.last_speed = speed self.speed_window = deque(maxlen=100) self.current_speed = 0 def end(self): if self.send_logs: self.experiment.end()
def run_exp(self): ''' Run the experiment on given number of pilots. ''' # Pre-train pretrain_weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, '_all') if self.pretraining: print('Pretraining...') _ = self.pretrain(pretrain_weights_filename, self.patience) for pilot_idx in range(1, self.n_pilots + 1): if self.log_pilots: experiment = Experiment(api_key='cSZq9kuH2I87ezvm2dEWTx6op', project_name='pilot{}'.format(pilot_idx), log_code=False, auto_param_logging=False) else: experiment = None # Load pilot data X_train, y_train, X_valid, y_valid, X_test, y_test = self.load_data(pilot_idx, valid_ratio=0.2) # Construct model & load pre-trained weights if available weights_filename = '{}{}.h5'.format(self.weights_savename_prefix, pilot_idx) self.model = self.build_model(load_weights=True if self.pretraining else False, weights_filename=pretrain_weights_filename) # Train print('First phase training - Pilot {}'.format(pilot_idx)) hist, best_epoch = self.train(X_train, y_train, X_valid, y_valid, save_filename='{}{}.h5'.format(self.weights_savename_prefix, pilot_idx), patience=self.patience, experiment=experiment) self.histories.append(hist) # Test (before extra training) self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, False) # Extra-train if self.train_after_es: hist_es = self.extra_train(pilot_idx, X_valid, y_valid, weights_filename, hist.history['loss'][best_epoch], self.max_after_es_epochs) # New self.histories_es.append(hist_es) # Test (after extra training) self.test(pilot_idx, weights_filename, X_train, y_train, X_test, y_test, True) if self.log_pilots: experiment.log_metrics({'Test accuracy' : self.test_score['accuracy'][-1]}) # Get t-SNE from intermediary outputs layer_idxs = self.tsne_layer_idxs get_output_functions = [ K.function([self.model.layers[0].input], [self.model.layers[idx].output]) for idx in layer_idxs] # Training dataset layer_outputs = [ get_output([X_train])[0] for get_output in get_output_functions ] [ experiment.log_figure(figure_name='tsne_raw_train{}_layer{}'.format(pilot_idx, layer_idxs[idx]), figure=tsne_plot(layer_outputs[idx], y_train, 20, title="t-SNE - Pilot {} - Layer {} (train)".format(pilot_idx, layer_idxs[idx]))) for idx in range(len(layer_idxs)) ] # Testing dataset layer_outputs = [ get_output([X_test])[0] for get_output in get_output_functions ] [ experiment.log_figure(figure_name='tsne_raw_test{}_layer{}'.format(pilot_idx, layer_idxs[idx]), figure=tsne_plot(layer_outputs[idx], y_test, 20, title="t-SNE - Pilot {} - Layer {} (test)".format(pilot_idx, layer_idxs[idx]))) for idx in range(len(layer_idxs)) ] plt.close('all') experiment.end() # Export logs to Comet.ml if self.comet_log: self.log() return self.test_score
def victim(kwargs=None): def comet_pull_poison(craftstep): for attempt in range(5): try: bytefile = craftexpt.get_asset(assets[craftstep]) if localrank == 0: print('==> poisoninputs-{} pulled'.format(craftstep)) poisoninputs = pickle.loads(bytefile) return poisoninputs[:args.npoison] except: print(f'WARNING: comet pull attempt for craftstep {craftstep} failed on attempt {attempt}') sleep(5) if kwargs is not None: for key in kwargs: globals()[key] = kwargs[key] for key in argsmod: setattr(args, key, argsmod[key]) craftexpt = api.get_experiment(cometconfig["workspace"], args.craftproj, args.craftkey) assets = {asset['step']: asset['assetId'] for asset in craftexpt.get_asset_list() if 'poisoninputs-' in asset['fileName']} print('==> begin victim train') trial = 0 while args.ntrial is None or trial < args.ntrial: for craftstep in args.craftsteps: experiment = Experiment(project_name=args.victimproj, auto_param_logging=False, auto_metric_logging=False, parse_args=False) experiment.log_parameters(vars(args)) experiment.set_name(f'{args.craftkey[:5]}-{experiment.get_key()[:5]}') experiment.add_tag(args.tag) # experiment.add_tag(args.Xtag) experiment.log_parameters(dict(craftstep=craftstep, trial=trial)) experiment.log_other('crafturl', craftexpt.url) experiment.log_other('command', 'python ' + ' '.join(sys.argv)) if localrank == 0: print_command_and_args(args); print('crafturl: ' + craftexpt.url) if 'victim.py' in sys.argv[0]: poisoninputs = comet_pull_poison(craftstep) if poisoninputs is None: experiment.end(); print(f'skipping craftstep {craftstep}'); continue if args.savepoisondataset: package_poisoned_dataset(poisoninputs, xtrain, ytrain, xtarget, ytarget, ytargetadv, xvalid, yvalid, args, craftstep); experiment.end(); continue # meta.init_weights(sess, pretrain_weights) # what we had before meta.global_initialize(args, sess) meta.poisoninputs.load(poisoninputs, sess) trainstep = 0 for epoch in range(args.nvictimepoch): tic = time() lrnrate = lr_schedule(args.lrnrate, epoch, args.warmupperiod, args.schedule) # log hidden layer features if args.logfeat and epoch == args.nvictimepoch - 1: feats = [] for victimfeed in feeddict_generator(xtrain, ytrain, lrnrate, meta, args, victim=True): hiddens = sess.run(meta.hiddens, victimfeed) for i, hidden in enumerate(hiddens): if len(feats) <= i: feats.append(defaultdict(list)) feat = np.reshape(hidden, [-1, np.prod(hidden.shape[1:])]) appendfeats(feats[i], feat, victimfeed, ybase, ytarget, args.batchsize) for i, feats_layer in enumerate(feats): comet_log_asset(experiment, f'feats_layer{i}', feats_layer, step=epoch) # log validation acc if epoch in np.round((args.nvictimepoch - 1) * np.linspace(0, 1, args.nvalidpoints) ** 2): resVs = [] # validation for _, validfeed, _ in feeddict_generator(xvalid, yvalid, lrnrate, meta, args, valid=True): resV = sess.run(meta.resultV, validfeed) resVs.append(resV) experiment.log_metrics(avg_n_dicts(resVs), step=trainstep) # train one epoch for victimfeed in feeddict_generator(xtrain, ytrain, lrnrate, meta, args, victim=True): _, resL = sess.run([meta.trainop, meta.resultL,], victimfeed) if not trainstep % 200: experiment.log_metrics(resL, step=trainstep) trainstep += 1 experiment.log_metric('elapsed', time() - tic, step=trainstep) if args.saveweights: comet_log_asset_weights_and_buffers(epoch, experiment, meta, sess) if not epoch % 20 and localrank == 0: print(' | '.join([f'{args.craftkey[:5]}-{args.tag} | trial-{trial} | craftstep-{craftstep} | epoch {epoch} | elapsed {round(time() - tic, 2)}'] + [f'{key} {trunc_decimal(val)}' for key, val in resL.items() if 'class' not in key] + [f'{key} {trunc_decimal(val)}' for key, val in resV.items() if 'class' not in key])) experiment.end() trial += 1
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', type=str, default='gcn') parser.add_argument('--type', type=str, default='base') parser.add_argument('--runs', type=int, default=10) parser.add_argument('--epochs', type=int, default=200) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--hidden_dim', type=int, default=300) parser.add_argument('--num_layers', type=int, default=5) args = parser.parse_args() assert args.model in ['gcn', 'sage', 'gin'] assert args.type in [ 'base', 'transfer', 'transfer-damaged', 'self-transfer', 'self-transfer-damaged' ] assert args.runs >= 1 assert args.epochs >= 1 assert args.lr > 0 assert args.hidden_dim > 0 assert args.num_layers > 0 # --------------------------------------------------- # MODEL # --------------------------------------------------- model = networks[args.model](in_channels=dataset.num_features, hidden_channels=args.hidden_dim, out_channels=dataset.num_tasks, num_conv_layers=args.num_layers).to(device) # --------------------------------------------------- # EXPERIMENT DETAILS # --------------------------------------------------- print('Graph Classification Experiment') print('Mol_HIV task') print(exp_description[args.type]) print('----------------------------------------------') print('Model: {}'.format(args.model)) print('Number of runs: {}'.format(args.runs)) print('Number of epochs: {}'.format(args.epochs)) print('Learning rate: {}'.format(args.lr)) print() print(model) # --------------------------------------------------- # EXPERIMENT LOOP # --------------------------------------------------- for run in range(args.runs): print() print('Run #{}'.format(run + 1)) # Model initialisation if args.type == 'base': model.reset_parameters() elif args.type in ['transfer', 'transfer-damaged']: # Pretrain on Mol-BBBP model.reset_parameters() bbbp_optimiser = optim.Adam(model.parameters(), lr=0.001) to_damage = args.type == 'transfer-damaged' print('Pretraining model on Mol-BBBP...') best_val_acc = pretrain_molbbbp(model, device, bbbp_evaluator, bbbp_optimiser, args.model, damage=to_damage) print('Validation accuracy: {:.3}'.format(best_val_acc)) model.load_state_dict( torch.load('molbbbp_models/{}_molbbbp.pth'.format(args.model))) elif args.type in ['self-transfer', 'self-transfer-damaged']: # Pretrain on Mol-HIV Source Split model.reset_parameters() source_optimiser = optim.Adam(model.parameters(), lr=0.001) to_damage = args.type == 'self-transfer-damaged' print('Pretraining model on Mol-HIV Source Task...') best_val_acc = pretrain_source_molhiv(model, device, evaluator, source_optimiser, args.model, damage=to_damage) print('Validation accuracy: {:.3}'.format(best_val_acc)) model.load_state_dict( torch.load('molhiv/{}_source_molhiv.pth'.format(args.model))) # Comet Experiment experiment = Experiment(project_name='graph-classification', display_summary_level=0, auto_metric_logging=False) experiment.add_tags([args.model, args.type]) experiment.log_parameters({ 'hidden_dim': args.hidden_dim, 'num_features': dataset.num_features, 'num_classes': dataset.num_tasks, 'learning_rate': args.lr, 'num_epochs': args.epochs, }) # Mol-HIV Target Training print('Training on Mol-HIV') optimizer = optim.Adam(model.parameters(), args.lr) for epoch in tqdm(range(args.epochs)): train_loss = train(model, device, target_loader, optimizer) train_performance = eval(model, device, target_loader, evaluator) experiment.log_metric('train_loss', train_loss.item(), step=epoch) experiment.log_metric('train_roc-auc', train_performance[dataset.eval_metric], step=epoch) experiment.end()
def comet_lgbm(save_path): from comet_ml import Experiment exp = Experiment(api_key="sqMrI9jc8kzJYobRXRuptF5Tj", project_name="baseline", workspace="gdreiman1") exp.log_code = True import pickle import pandas as pd import lightgbm as lgb import numpy as np import sklearn import matplotlib.pyplot as plt from sklearn.metrics import precision_recall_fscore_support as prf #%% def single_roc(y_preds,y_true): from sklearn.metrics import roc_curve, auc,precision_recall_curve fpr, tpr, _ = roc_curve(y_true, y_preds) roc_auc = auc(fpr, tpr) plt.figure() lw = 2 plt.plot(fpr, tpr, color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.title('Receiver operating characteristic example') precision, recall, thresholds = precision_recall_curve(y_true, y_preds) plt.plot(recall, precision, color='blue', lw=lw, label='Precision vs Recall') # show the plot plt.legend(loc="lower right") plt.show() def multi_roc(y_preds,y_true,name,n_classes): import collections nested_dict = lambda: collections.defaultdict(nested_dict) data_store = nested_dict() from sklearn.metrics import roc_curve, auc from scipy import interp from itertools import cycle lw = 2 name_store = ['Active', 'Inactive', 'Inconclusive'] fpr = dict() tpr = dict() roc_auc = dict() for i in range(n_classes): fpr[i], tpr[i], _ = roc_curve(y_true[:, i], y_preds[:, i]) roc_auc[i] = auc(fpr[i], tpr[i]) # Compute micro-average ROC curve and ROC area fpr["micro"], tpr["micro"], _ = roc_curve(y_true[:, i].ravel(), y_preds[:, i].ravel()) roc_auc["micro"] = auc(fpr["micro"], tpr["micro"]) # Compute macro-average ROC curve and ROC area # First aggregate all false positive rates all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)])) # Then interpolate all ROC curves at this points mean_tpr = np.zeros_like(all_fpr) for i in range(n_classes): mean_tpr += interp(all_fpr, fpr[i], tpr[i]) # Finally average it and compute AUC mean_tpr /= n_classes fpr["macro"] = all_fpr tpr["macro"] = mean_tpr roc_auc["macro"] = auc(fpr["macro"], tpr["macro"]) # Plot all ROC curves plt.figure() plt.plot(fpr["micro"], tpr["micro"], label='micro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["micro"]), color='deeppink', linestyle=':', linewidth=4) plt.plot(fpr["macro"], tpr["macro"], label='macro-average ROC curve (area = {0:0.2f})' ''.format(roc_auc["macro"]), color='navy', linestyle=':', linewidth=4) colors = cycle(['aqua', 'darkorange', 'cornflowerblue','green']) for i, color in zip(range(n_classes), colors): plt.plot(fpr[i], tpr[i], color=color, lw=lw, label='ROC curve of '+ name_store[i]+'(area = {1:0.2f})' ''.format(i, roc_auc[i])) plt.plot([0, 1], [0, 1], 'k--', lw=lw) plt.xlim([0.0, 1.0]) plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') #plt.title('Multi-class ROC for '+name+' Split= '+str(count+1)) plt.title('Multi-class ROC for '+name) plt.legend(loc="lower right") #plt.show() #%% #save_path = r'C:\Users\gdrei\Dropbox\UCL\Thesis\May_13\AID_1345083_processed.pkl' model_type = 'lgbm' #get data cleaned pickle_off = open(save_path,'rb') activity_table=pickle.load(pickle_off) pickle_off.close() #get length of MFP fp_length = len(activity_table.iloc[5]['MFP']) from sklearn.preprocessing import StandardScaler, LabelEncoder scaler = StandardScaler(copy = False) le = LabelEncoder() labels = le.fit_transform(activity_table['PUBCHEM_ACTIVITY_OUTCOME']) #split data: from sklearn.model_selection import StratifiedShuffleSplit splitter = StratifiedShuffleSplit(n_splits=1, test_size=0.5, train_size=None, random_state=2562) X_mfp = np.concatenate(np.array(activity_table['MFP'])).ravel() X_mfp = X_mfp.reshape((-1,fp_length)) for train_ind, test_ind in splitter.split(X_mfp,labels): # standardize data X_train_molchars_std = scaler.fit_transform(np.array(activity_table.iloc[train_ind,4:])) X_test_molchars_std = scaler.transform(np.array(activity_table.iloc[test_ind,4:])) X_train = np.concatenate((X_mfp[train_ind,:],X_train_molchars_std),axis = 1) X_test = np.concatenate((X_mfp[test_ind,:],X_test_molchars_std),axis = 1) y_train = labels[train_ind] y_test = labels[test_ind] #X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X,labels,test_size = .5, shuffle = True, stratify = labels, random_state = 2562) bin_y_train, bin_y_test = [1 if x ==2 else x for x in y_train],[1 if x ==2 else x for x in y_test] #do light gbm #need to make a lib svm file train_data = lgb.Dataset(X_train,label=y_train) test_data = lgb.Dataset(X_test,label=y_test) #make model class lgbm_model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=-1, learning_rate=0.1, n_estimators=500, subsample_for_bin=200000, objective='binary', is_unbalance=True, min_split_gain=0.0, min_child_weight=0.001, min_child_samples=20, subsample=1.0, subsample_freq=0, colsample_bytree=1.0, reg_alpha=0.0, reg_lambda=0.0, random_state=None, n_jobs=-1, silent=True, importance_type='split') #train model trained_mod = lgbm_model.fit(X_train,y_train) #predict classes and class_probs test_class_preds = lgbm_model.predict(X_test) test_prob_preds = lgbm_model.predict_proba(X_test) #calculate Class report class_rep = sklearn.metrics.classification_report(y_test,test_class_preds) print(class_rep) if len(set(y_test)) == 2: single_roc(test_prob_preds[:,1],y_test) prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None) else: from tensorflow.keras.utils import to_categorical multi_roc(test_prob_preds,to_categorical(y_test),'',3) prec,rec,f_1,supp = prf(y_test, test_class_preds, average=None) #%% '''Comet Saving Zone''' #get AID number import ntpath #get base file name folder,base = ntpath.split(save_path) #split file name at second _ assumes file save in AID_xxx_endinfo.pkl AID, _,end_info = base.rpartition('_') #save data location, AID info, and version info exp.log_dataset_info(name = AID, version = end_info, path = save_path) #save model params exp.log_parameters(trained_mod.get_params()) #save metrics report to comet if len(f_1) == 2: for i,name in enumerate(['Active','Inactive']): exp.log_metric('f1 class '+name, f_1[i]) exp.log_metric('Recall class'+name,rec[i]) exp.log_metric('Precision class'+name, prec[i]) else: for i,name in enumerate(['Active','Inconclusive','Inactive']): exp.log_metric('f1 class '+str(i), f_1[i]) exp.log_metric('Recall class'+str(i),rec[i]) exp.log_metric('Precision class'+str(i), prec[i]) #exp.log_metric('f1 class '+str(i), f_1[i]) #exp.log_metric('Recall class'+str(i),rec[i]) #exp.log_metric('Precision class'+str(i), prec[i]) exp.log_other('Classification Report',class_rep) #save model in data_folder with comet experiement number associated exp_num = exp.get_key() model_save = folder+'\\'+model_type+'_'+exp_num+'.pkl' pickle_on = open(model_save,'wb') pickle.dump(trained_mod,pickle_on) pickle_on.close() #log trained model location exp.log_other('Trained Model Path',model_save) #save some informatvie tags: tags = [AID,end_info,model_type] exp.add_tags(tags) #save ROC curve exp.log_figure(figure_name = 'ROC-Pres/Recall',figure=plt) plt.show() #tell comet that the experiement is over exp.end()
def startExperiment(self): experiment = Experiment(api_key=self.API_KEY, project_name=self.comet_project_name, workspace=self.WORKSPACE, log_code=True) # Disable cuda support if not self.cuda_support: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' print(self.experiment_name) features_temp = [] if self.feature_name.name == FeatureEnums.mfcc.name: self.features = DataGrabber.getMfccs5Seconds_128_256_64() self.model_with_maxpool = True elif self.feature_name.name == FeatureEnums.env.name: self.features = DataGrabber.getAmplitude10() elif self.feature_name.name == FeatureEnums.zero_crossing.name: self.features = DataGrabber.getZeroCrossingRate256_64() elif self.feature_name.name == FeatureEnums.chroma_stft.name: self.features = DataGrabber.getChromaStft128_256_64() self.model_with_maxpool = True elif self.feature_name.name == FeatureEnums.chroma_cqt.name: self.features = DataGrabber.getChromaCqt5Sec128_7_64() self.model_with_maxpool = True elif self.feature_name.name == FeatureEnums.rms.name: self.features = DataGrabber.getRMS() dir_files = os.listdir(self.DATASET_DIRECTORY_PATH) dir_files.sort(key=lambda f: int(re.sub('\D', '', f))) para_count = 0 for file in dir_files: if "xml" in file: sample_number = int(file.split(".")[0][4:]) tree = ET.parse( os.path.join(os.path.abspath(self.DATASET_DIRECTORY_PATH), file)) root = tree.getroot() class_labels = [] if self.vst_parameter is None: for x in range(99): class_labels.append( int(root[x + 2].attrib["presetValue"])) para_count = 99 else: para_count = len(self.vst_parameter) for para in self.vst_parameter: class_labels.append( int(root[para + 2].attrib["presetValue"])) data = self.features[sample_number] features_temp.append([data, class_labels]) del dir_files, self.features, tree, root print("All Data Appended") # Convert into a Panda dataframe features_panda = pd.DataFrame(features_temp, columns=['feature', 'class_label']) del features_temp # Convert features and corresponding classification labels into numpy arrays feature_array = np.asarray(features_panda.feature.tolist()) value_array = np.asarray(features_panda.class_label.tolist()) del features_panda le = LabelEncoder() value_array = to_categorical(le.fit_transform(value_array)) # split the dataset from sklearn.model_selection import train_test_split x_train, x_test, y_train, y_test = train_test_split(feature_array, value_array, test_size=0.2, random_state=42) if self.model_with_maxpool: train_shape = (len(x_train[1]), len(x_train[1][1])) model = self.createModel2D(train_shape, para_count) else: train_shape = len(x_train[1]) model = self.createModel1D(train_shape, para_count) del feature_array, value_array start = datetime.now() # preparing callbacks my_callbacks = [ # stops training after 20 epoch without improvement tf.keras.callbacks.EarlyStopping(patience=20), # saves model after each epoch tf.keras.callbacks.ModelCheckpoint( filepath=self.checkpoint_name + "_" + self.feature_name.name + str(self.count_first_layer) + str(self.count_second_layer) + str(self.count_third_layer) + datetime.now().strftime("%m%d%Y%H%M%S") + ".{epoch:02d}-{val_loss:.4f}.h5"), # reduce learning rate after 5 epochs without improvement tf.keras.callbacks.ReduceLROnPlateau(monitor="loss", factor=0.5, patience=5, min_lr=0.0001, verbose=1) ] print(model.summary()) print("start Training....") history = model.fit( x_train, y_train, batch_size=self.batch_size, epochs=self.num_epochs, validation_data=(x_test, y_test), verbose=2, callbacks=my_callbacks) # validation_data=(x_test, y_test) duration = datetime.now() - start print("Training completed in time: ", duration) print("used feature: {}".format(self.feature_name)) experiment.end() score = model.evaluate(x_train, y_train, verbose=0) print("train accuracy: {}".format(score)) # experiment.log_metric("train_acc", score) score = model.evaluate(x_test, y_test, verbose=0) print('test accuracy: {}'.format(score))
class CometWriter(Writer): ''' A Writer object to be used by all.experiments.Experiment. Writes logs using comet.ml Requires an API key to be stored in .comet.config or as an environment variable. Look at https://www.comet.ml/docs/python-sdk/advanced/#python-configuration for more info. Args: experiment (all.experiments.Experiment): The Experiment associated with the Writer object. agent_name (str): The name of the Agent the Experiment is being performed on env_name (str): The name of the environment the Experiment is being performed in loss (bool, optional): Whether or not to log loss/scheduling metrics, or only evaluation and summary metrics. logdir (str): The directory where run information is stored. ''' def __init__(self, experiment, agent_name, env_name, loss=True, logdir='runs'): self.env_name = env_name self._experiment = experiment self._loss = loss try: from comet_ml import Experiment except ImportError as e: print("Failed to import comet_ml. CometWriter requires that comet_ml be installed") raise e try: self._comet = Experiment(project_name=env_name) except ImportError as e: print("See https://www.comet.ml/docs/python-sdk/warnings-errors/ for more info on this error.") raise e except ValueError as e: print("See https://www.comet.ml/docs/python-sdk/advanced/#python-configuration for more info on this error.") raise e self._comet.set_name(agent_name) self.log_dir = logdir def add_loss(self, name, value, step="frame"): if self._loss: self.add_evaluation("loss/" + name, value, step) def add_evaluation(self, name, value, step="frame"): self._comet.log_metric(name, value, self._get_step(step)) def add_schedule(self, name, value, step="frame"): if self._loss: self.add_scalar(name, value, step) def add_summary(self, name, mean, std, step="frame"): self.add_evaluation(name + "/mean", mean, step) self.add_evaluation(name + "/std", std, step) def add_scalar(self, name, value, step="frame"): self._comet.log_metric(name, value, self._get_step(step)) def _get_step(self, _type): if _type == "frame": return self._experiment.frame if _type == "episode": return self._experiment.episode return _type def close(self): self._comet.end()
def train(args, use_comet: bool = True): data_cls = funcs[args['dataset']] model_cls = funcs[args['model']] network = funcs[args['network']] print('[INFO] Getting dataset...') data = data_cls() (x_train, y_train), (x_test, y_test) = data.load_data() classes = data.mapping # #Used for testing only # x_train = x_train[:100, :, :] # y_train = y_train[:100, :] # x_test = x_test[:100, :, :] # y_test = y_test[:100, :] # print ('[INFO] Training shape: ', x_train.shape, y_train.shape) # print ('[INFO] Test shape: ', x_test.shape, y_test.shape) # #delete these lines y_test_labels = [ np.where(y_test[idx] == 1)[0][0] for idx in range(len(y_test)) ] # distribute 90% test 10% val dataset with equal class distribution (x_test, x_valid, y_test, y_valid) = train_test_split(x_test, y_test, test_size=0.1, stratify=y_test_labels, random_state=42) print('[INFO] Training shape: ', x_train.shape, y_train.shape) print('[INFO] Validation shape: ', x_valid.shape, y_valid.shape) print('[INFO] Test shape: ', x_test.shape, y_test.shape) print('[INFO] Setting up the model..') model = model_cls(network, data_cls) print(model) dataset = dict({ 'x_train': x_train, 'y_train': y_train, 'x_valid': x_valid, 'y_valid': y_valid, 'x_test': x_test, 'y_test': y_test }) if use_comet and args['find_lr'] == False: #create an experiment with your api key experiment = Experiment(api_key='INSERT API KEY', project_name='emnist', auto_param_logging=False) print('[INFO] Starting Training...') #will log metrics with the prefix 'train_' with experiment.train(): _ = train_model(model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], name=args['network']) print('[INFO] Starting Testing...') #will log metrics with the prefix 'test_' with experiment.test(): loss, score = model.evaluate(dataset, args['batch_size']) print(f'[INFO] Test evaluation: {score*100}') metrics = {'loss': loss, 'accuracy': score} experiment.log_metrics(metrics) experiment.log_parameters(args) experiment.log_dataset_hash( x_train) #creates and logs a hash of your data experiment.end() elif use_comet and args['find_lr'] == True: _ = train_model(model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], FIND_LR=args['find_lr'], name=args['network']) else: print('[INFO] Starting Training...') train_model(model, dataset, batch_size=args['batch_size'], epochs=args['epochs'], name=args['network']) print('[INFO] Starting Testing...') loss, score = model.evaluate(dataset, args['batch_size']) print(f'[INFO] Test evaluation: {score*100}') if args['weights']: model.save_weights() if args['save_model']: model.save_model()
## Experiment 1: everything as normal, using .train(): experiment = Experiment() experiment.add_tag("metrics") results = {} gbm = xgb.train( params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=True, evals_result=results, ) experiment.end() ## Experiment 2: no results (thus no metrics), using .train(): experiment = Experiment() experiment.add_tag("no metrics") gbm = xgb.train( params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=True, ) experiment.end()
def cli_main(): parser = options.get_training_parser() parser.add_argument( "--comet-logging", action="store_true", help="Whether to use Comet.ML for logging", ) args = options.parse_args_and_arch(parser) logging = getattr(args, "comet_logging", False) config = None if logging: PROJECT = "machine-translation" if not keyring.get_password("comet", PROJECT): comet_ml_api_key = getpass("Please enter the comet.ml API key: ") keyring.set_password("comet", PROJECT, comet_ml_api_key) else: comet_ml_api_key = keyring.get_password("comet", PROJECT) experiment = Experiment( api_key=comet_ml_api_key, project_name="machine-translation", workspace="machine-translation", auto_output_logging=None, ) config = { "api_key": comet_ml_api_key, "experiment_key": experiment.get_key() } print("Proceeding with Comet.ML logging...") if args.distributed_init_method is None: distributed_utils.infer_init_method(args) if args.distributed_init_method is not None: # distributed training if torch.cuda.device_count() > 1 and not args.distributed_no_spawn: start_rank = args.distributed_rank args.distributed_rank = None # assign automatically torch.multiprocessing.spawn( fn=distributed_main, args=(args, config, start_rank), nprocs=torch.cuda.device_count(), ) else: distributed_main(args.device_id, args, config) elif args.distributed_world_size > 1: # fallback for single node with multiple GPUs assert args.distributed_world_size <= torch.cuda.device_count() port = random.randint(10000, 20000) args.distributed_init_method = "tcp://localhost:{port}".format( port=port) args.distributed_rank = None # set based on device id if max(args.update_freq) > 1 and args.ddp_backend != "no_c10d": print( "| NOTE: you may get better performance with: --ddp-backend=no_c10d" ) torch.multiprocessing.spawn(fn=distributed_main, args=(args, config), nprocs=args.distributed_world_size) else: # single GPU training main(args, config=config) if config: experiment.end()
return start_indexs def getNextIterInds(firstPicksList, fplist, bottom_to_select): diverse_picks = mmp.LazyBitVectorPick( fplist, len(fplist), len(firstPicksList) + bottom_to_select, firstPicksList) start_indexs = np.array(diverse_picks) return start_indexs with parallel_backend('multiprocessing'): if selection_type == 'Diverse': start_ind_list = Parallel(n_jobs=5)( delayed(getNextIterInds)( firstPicksList=i, fplist=j, bottom_to_select=k) for i, j, k in zip(start_ind_list, fp_metalist, diverse_size_list)) elif selection_type == 'Random': start_ind_list = Parallel(n_jobs=5)( delayed(getRandomIterInds)( firstPicksList=i, fplist=j, bottom_to_select=k) for i, j, k in zip(start_ind_list, fp_metalist, diverse_size_list)) iter_num += 1 metrics_df = pd.DataFrame(metric_dict_list) multi_dump_path = os.path.join('/home/gabriel/Dropbox/UCL/Thesis/Data/', 'second_diverse_GCNN_50epoch_iter_run.pkl') exp.log_other('Metrics Dict Path', multi_dump_path) metrics_df.to_pickle(multi_dump_path) exp.end()
def train(self, train_data, cometml_key=None): if cometml_key is not None: experiment = Experiment(api_key=cometml_key, project_name="dsgym-tgan", workspace="baukebrenninkmeijer") experiment.log_parameter('batch_size', self.batch_size) experiment.log_parameter('embeddingDim', self.embeddingDim) experiment.log_parameter('genDim', self.genDim) experiment.log_parameter('disDim', self.disDim) experiment.log_parameter('GAN version', 'TGAN') # writer = SummaryWriter() # train_data = monkey_with_train_data(train_data) print('Transforming data...') self.transformer = BGMTransformer(self.meta) self.transformer.fit(train_data) pickle.dump(self.transformer, open(f'{self.working_dir}/transformer.pkl', 'wb')) train_data = self.transformer.transform(train_data) # ncp1 = sum(self.transformer.components[0]) # ncp2 = sum(self.transformer.components[1]) # for i in range(ncp1): # for j in range(ncp2): # cond1 = train_data[:, 1 + i] > 0 # cond2 = train_data[:, 2 + ncp1 + j] # cond = np.logical_and(cond1, cond2) # # mean1 = train_data[cond, 0].mean() # mean2 = train_data[cond, 1 + ncp1].mean() # # std1 = train_data[cond, 0].std() # std2 = train_data[cond, 1 + ncp1].std() # print(i, j, np.sum(cond), mean1, std1, mean2, std2, sep='\t') # dataset = torch.utils.data.TensorDataset(torch.from_numpy(train_data.astype('float32')).to(self.device)) # loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, shuffle=True, drop_last=True) data_sampler = Sampler(train_data, self.transformer.output_info) data_dim = self.transformer.output_dim self.cond_generator = Cond(train_data, self.transformer.output_info) self.generator = Generator( self.embeddingDim + self.cond_generator.n_opt, self.genDim, data_dim).to(self.device) self.discriminator = Discriminator( data_dim + self.cond_generator.n_opt, self.disDim).to(self.device) optimizerG = optim.Adam(self.generator.parameters(), lr=2e-4, betas=(0.5, 0.9), weight_decay=self.l2scale) optimizerD = optim.Adam(self.discriminator.parameters(), lr=2e-4, betas=(0.5, 0.9)) #, weight_decay=self.l2scale) # pickle.dump(self, open(f'{self.working_dir}/tgan_synthesizer.pkl', 'wb')) # writer.add_graph(self.generator) max_epoch = max(self.store_epoch) assert self.batch_size % 2 == 0 mean = torch.zeros(self.batch_size, self.embeddingDim, device=self.device) std = mean + 1 print('Starting training loop...') steps_per_epoch = len(train_data) // self.batch_size for i in tqdm(range(max_epoch)): for id_ in tqdm(range(steps_per_epoch), leave=False): fakez = torch.normal(mean=mean, std=std) condvec = self.cond_generator.generate(self.batch_size) if condvec is None: c1, m1, col, opt = None, None, None, None real = data_sampler.sample(self.batch_size, col, opt) else: c1, m1, col, opt = condvec c1 = torch.from_numpy(c1).to(self.device) m1 = torch.from_numpy(m1).to(self.device) fakez = torch.cat([fakez, c1], dim=1) perm = np.arange(self.batch_size) np.random.shuffle(perm) real = data_sampler.sample(self.batch_size, col[perm], opt[perm]) c2 = c1[perm] fake = self.generator(fakez) fakeact = apply_activate(fake, self.transformer.output_info) real = torch.from_numpy(real.astype('float32')).to(self.device) if c1 is not None: fake_cat = torch.cat([fakeact, c1], dim=1) real_cat = torch.cat([real, c2], dim=1) else: real_cat = real fake_cat = fake # print(real_cat[0]) # print(fake_cat[0]) # assert 0 y_fake = self.discriminator(fake_cat) y_real = self.discriminator(real_cat) # loss_d = -(torch.log(torch.sigmoid(y_real) + 1e-4).mean()) - (torch.log(1. - torch.sigmoid(y_fake) + 1e-4).mean()) loss_d = -(torch.mean(y_real) - torch.mean(y_fake)) pen = calc_gradient_penalty(self.discriminator, real_cat, fake_cat, self.device) optimizerD.zero_grad() pen.backward(retain_graph=True) loss_d.backward() optimizerD.step() # for p in discriminator.parameters(): # p.data.clamp_(-0.05, 0.05) fakez = torch.normal(mean=mean, std=std) condvec = self.cond_generator.generate(self.batch_size) if condvec is None: c1, m1, col, opt = None, None, None, None else: c1, m1, col, opt = condvec c1 = torch.from_numpy(c1).to(self.device) m1 = torch.from_numpy(m1).to(self.device) fakez = torch.cat([fakez, c1], dim=1) fake = self.generator(fakez) fakeact = apply_activate(fake, self.transformer.output_info) if c1 is not None: y_fake = self.discriminator(torch.cat([fakeact, c1], dim=1)) else: y_fake = self.discriminator(fakeact) if condvec is None: cross_entropy = 0 else: cross_entropy = cond_loss(fake, self.transformer.output_info, c1, m1) # loss_g = -torch.log(torch.sigmoid(y_fake) + 1e-4).mean() + cross_entropy loss_g = -torch.mean(y_fake) + cross_entropy optimizerG.zero_grad() loss_g.backward() optimizerG.step() if cometml_key: experiment.log_metric('Discriminator Loss', loss_d) experiment.log_metric('Generator Loss', loss_g) # print("---") # print(fakeact[:, 0].mean(), fakeact[:, 0].std()) # print(fakeact[:, 1 + ncp1].mean(), fakeact[:, 1 + ncp1].std()) print(i + 1, loss_d.data, pen.data, loss_g.data, cross_entropy) if cometml_key: experiment.log_epoch_end(i) if i + 1 in self.store_epoch: print('Saving model') torch.save( { "generator": self.generator.state_dict(), "discriminator": self.discriminator.state_dict(), }, "{}/model_{}.tar".format(self.working_dir, i + 1)) if cometml_key is not None: experiment.end()