def pretraining(deepforest, BASE_PATH): # import comet_ml logger comet_experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2", project_name="deepforest", workspace="bw4sz") comet_experiment.log_parameters(deepforest_model.config) comet_experiment.log_parameter("Type","Pretraining") comet_experiment.log_parameter("timestamp",timestamp) # create a dir for the run save_path = BASE_PATH + "snapshots/{}/".format(timestamp) os.mkdir(save_path) deepforest_model.config["save_path"] = save_path deepforest_model.config["snapshot_path"] = save_path list_of_tfrecords = glob.glob(BASE_PATH + "pretraining/tfrecords/*.tfrecord") deepforest_model.train(annotations=BASE_PATH + "pretraining/crops/pretraining.csv", input_type="tfrecord", list_of_tfrecords=list_of_tfrecords, comet_experiment=comet_experiment) if not deepforest_model.config["validation_annotations"] == "None": mAP = deepforest_model.evaluate_generator(annotations = deepforest_model.config["validation_annotations"], comet_experiment=comet_experiment) comet_experiment.log_metric("mAP", mAP) #retrain model based on hand annotation crops, assign the weights from pretraining model, multi-gpu model weights are split. deepforest_model.model.save_weights(BASE_PATH + "snapshots/pretraining_weights_{}.h5".format(timestamp)) deepforest_model.config["weights"] = BASE_PATH + "snapshots/pretraining_weights_{}.h5".format(timestamp) return deepforest_model
def train(pretrain_model_path, proportion_data, DeepForest_config): ###Log experiments experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2", project_name='deeplidar', log_code=False) #make snapshot dir dirname = datetime.now().strftime("%Y%m%d_%H%M%S") experiment.log_parameter("Start Time", dirname) save_snapshot_path = DeepForest_config["save_snapshot_path"] + dirname os.mkdir(save_snapshot_path) ##Replace config file and experiment DeepForest_config["batch_size"] = 40 DeepForest_config["epochs"] = 40 experiment.log_parameter("mode", "ablation") DeepForest_config["evaluation_images"] = 0 #set training images, as a function of the number of training windows DeepForest_config["training_proportion"] = proportion_data experiment.log_parameters(DeepForest_config) #Create model model, training_model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=1, weights=pretrain_model_path, multi_gpu=2, freeze_backbone=False, nms_threshold=DeepForest_config["nms_threshold"], input_channels=DeepForest_config["input_channels"]) if not proportion_data == 0: #Run training, and pass comet experiment class #start training data = load_retraining_data(DeepForest_config) train_generator, validation_generator = create_h5_generators( data, DeepForest_config=DeepForest_config) #ensure directory created first; otherwise h5py will error after epoch. history = training_model.fit_generator( generator=train_generator, steps_per_epoch=train_generator.size() / DeepForest_config["batch_size"], epochs=DeepForest_config["epochs"], verbose=2, shuffle=False, workers=DeepForest_config["workers"], use_multiprocessing=DeepForest_config["use_multiprocessing"], max_queue_size=DeepForest_config["max_queue_size"]) num_trees = train_generator.total_trees else: num_trees = 0 #Log trees experiment.log_parameter("Number of Training Trees", num_trees) return prediction_model, num_trees
class CometTracker: def __init__(self, comet_params, run_params): self.experiment = Experiment(**comet_params) self.experiment.log_parameters(run_params) def track_metric(self, metric, value): self.experiment.log_metric(metric, value)
def main_eval(args): assert args.load_from is not None, '--load_from required in eval mode' logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) dataset_train, dataset_test, scaler = get_data(args) logging.info(f'evaluation mode. Level: {args.level}') device = torch.device( 'cuda:0') if torch.cuda.is_available() else torch.device('cpu') n_features = dataset_train.items.shape[1] generator, discriminator = get_models(args, n_features, device) experiment = Experiment(args.comet_api_key, project_name=args.comet_project_name, workspace=args.comet_workspace) experiment.log_parameters(vars(args)) load_model(Path(args.load_from), generator, discriminator, None, None, device) n_events = len(dataset_test) steps = (args.gan_test_ratio * n_events) // args.eval_batch_size evaluate_model(generator, experiment, dataset_test, args.eval_batch_size, steps, args, device, scaler, 0)
def train(x_train, y_train, x_test, y_test): exp = Experiment(project_name="perception", auto_histogram_gradient_logging=True) # log custom hyperparameters exp.log_parameters(params) # log any custom metric exp.log_metric('custom_metric', 0.95) # log a dataset hash exp.log_dataset_hash(x_train) # Define model model = build_model_graph(exp) model.fit( x_train, y_train, batch_size=exp.get_parameter('batch-size'), epochs=exp.get_parameter('epochs'), validation_data=(x_test, y_test), ) score = model.evaluate(x_test, y_test, verbose=0) logging.info("Score %s", score) # Finalize model includes the following calls # exp.log_confusion_matrix() # exp.log_image() # exp.log_histogram_3d() # exp.add_tag() # exp.log_model() utils.finalize_model(model, x_train, y_train, x_test, y_test, exp)
def main(config, comet=False): config = Config(config) # comet-ml setting if comet: experiment = Experiment(api_key=config.api_key, project_name=config.project_name, workspace=config.workspace) experiment.log_parameters(config) # device and dataset setting device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') dataset = MCDataset(config.root, config.dataset_name) data = dataset[0].to(device) config.num_nodes = dataset.num_nodes config.num_users = int(data.num_users) config.num_relations = dataset.num_relations # defines number of edge types # set and init model model = GAE(config, random_init).to(device) model.apply(init_xavier) # train if comet: trainer = Trainer(model, dataset, data, calc_rmse, config.epochs, config.lr, config.weight_decay, experiment) else: trainer = Trainer(model, dataset, data, calc_rmse, config.epochs, config.lr, config.weight_decay) trainer.iterate()
def main(): STARTTIME0 = time.strftime('run_%Y_%m_%d_%H_%M_%s') METRICS = [] for ts_size in [3000, 5000, 5600]: for iteration in range(10): _, _, X_train, X_test, y_train, y_test, _ = process_data( size=ts_size) experiment = Experiment(api_key=os.environ['COMET_API_KEY'], project_name='color-ml') experiment.log_parameters(PARAMETERS_MEDIAN) with experiment.train(): regressor_median = fit(X_train, y_train) metrics_dict = get_metrics_dict(regressor_median, X_test, y_test, experiment) metrics_dict['iteration'] = iteration metrics_dict['ts_size'] = ts_size METRICS.append(metrics_dict) df = pd.DataFrame(METRICS) df.to_csv('learningurve_' + STARTTIME0 + '.csv') experiment.log_asset('learningurve_' + STARTTIME0 + '.csv')
class Experiment(): def __init__(self, api_key=None, **kwargs): self._exp = None self._id = uuid4().hex if api_key: self._exp = CometExperiment(api_key, log_code=False, auto_param_logging=False, auto_metric_logging=False, **kwargs) self._id = self._exp.get_key() def log_metric(self, name, value, step=None, epoch=None): if self._exp: self._exp.log_metric(name, value, step, epoch) def log_epoch_end(self, epoch_cnt, step=None): if self._exp: self._exp.log_epoch_end(epoch_cnt, step=step) def log_parameters(self, hp): if self._exp: self._exp.log_parameters(flatten(hp, reducer='underscore')) @property def id(self): return self._id[:12]
class CometMLLogger: def __init__(self): self.experiment = Experiment(api_key="iU4f44llKnowZwmrEo9wfR2ch", project_name="general", workspace="yahyaalaamassoud", log_code=False, log_graph=False) def log_params(self, params: Dict[str, int]): self.experiment.log_parameters(params) def log_metric(self, metric_name, metric_val, step=None): self.experiment.log_metric(metric_name, metric_val, step=step) def log_metrics(self, metrics: Dict[str, float], step=None): self.experiment.log_metrics(metrics, step=step) def log_figure(self, figure_name: str, step: str): self.experiment.log_image(image_data=self.__savefig(), name=figure_name, step=step, overwrite=False) def __savefig(self): buf = io.BytesIO() plt.savefig(buf, format='png') buf.seek(0) return Image.open(io.BytesIO(buf.getvalue()))
def train_lgbm(x_train, x_val, y_train, y_val): # Make sure labels not one-hot experiment = Experiment( api_key="VNQSdbR1pw33EkuHbUsGUSZWr", project_name="piratesofthecaribbean", workspace="florpi", auto_param_logging=False, ) experiment.log_parameters(hyperparameters) clf = lgb.LGBMClassifier(**hyperparameters) clf.fit(X=x_train, y=y_train, eval_set=(x_val, y_val), eval_metric='AUC') y_pred = clf.predict(x_val) visualize.plot_confusion_matrix( y_val, y_pred, classes=LABELS, normalize=True, experiment=experiment, ) visualize.plot_confusion_matrix( y_val, y_pred, classes=LABELS, normalize=False, experiment=experiment, )
def get_experiment_objects(): args = CLIParser().parse_args() comet = Experiment(api_key="<your_key>", project_name="<your_project>", workspace="<your_workspace>", log_code=False, auto_param_logging=False, auto_metric_logging=False, disabled=args.no_comet, display_summary=False) if not args.continue_train: # else args.save is the directory from which assets are loaded to # continue training args.save = "{}-{}".format(args.save, time.strftime("%Y%m%d-%H%M%S")) create_exp_dir(args) comet.set_name(args.save.split('/')[-1]) comet.log_parameters(vars(args)) copy_assets(args, comet) last_state = None if args.continue_train: last_state = torch.load(os.path.join(args.save, 'state.pt')) continue_random(last_state, args.cuda) else: init_random(args.seed, args.cuda) return args, comet, last_state
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.add_tags(self.config.tags) experiment.disable_mp() experiment.log_parameters(self.config) self.callbacks.append(experiment.get_keras_callback())
def train(x_train, y_train, x_test, y_test): exp = Experiment(api_key="API_KEY", project_name="PROJECT", workspace="WORKSPACE") # log custom hyperparameters exp.log_parameters(params) # Define model model = build_model_graph(exp) model.fit( x_train, y_train, batch_size=exp.get_parameter('batch-size'), epochs=exp.get_parameter('epochs'), validation_data=(x_test, y_test), ) score = model.evaluate(x_test, y_test, verbose=0) logging.info("Score %s", score) # Finalize model includes the following calls # exp.log_confusion_matrix() # exp.log_image() # exp.log_histogram_3d() # exp.add_tag() # exp.log_model() utils.finalize_model(model, x_train, y_train, x_test, y_test, exp)
def prerun(args, run_dir=True, exp=True): if run_dir: p = Path(args["run_dir"]) if p.exists(): raise ValueError("Rundir exists, please remove.") p.mkdir() with open(Path(p, "args.json"), "w") as fp: json.dump(args, fp) if exp: experiment = Experiment( api_key="pJ6UYxQwjYoYbCmmutkqP66ni", project_name=args["comet_project"], workspace="mlippie", auto_metric_logging=True, auto_param_logging=False, log_graph=True, disabled="maximl" in socket.gethostname() # disable on dev machine ) experiment.log_parameters(args) return experiment else: return None
def __init__(self, experiment: Experiment, step: STEP, n_classes=2, topk: [int] = [1], class_map: List[str] = [], metrics=[]): """ :param experiment: :param step: :param n_classes: :param topk: :param class_map: :param metrics: """ assert len(class_map) == n_classes, \ f"Class map: {class_map} length is {len(class_map)} and is not equal to n_classes: {n_classes}" assert len(topk) != 0, \ f"topk param should be not empty" topk = np.sort(topk) assert topk[-1] <= n_classes, \ f"topk max class is {topk[-1]} which is greater than maximum class number: {n_classes}" self.n_classes = n_classes self.class_map = class_map experiment.log_parameters(dict(zip(class_map, [i for i in range(len(class_map))]))) self.step = step self.topk = topk self.experiment = experiment self.curr_state = {} self.metrics = ['ACC', 'LOSS'] + metrics self.accuracy_meter = torchnet.meter.ClassErrorMeter(topk=topk, accuracy=True) # accepts probs + labels self.ap_accuracy_meter = APMeter(n_classes) self.loss_meter = torchnet.meter.AverageValueMeter() self.other_meters: Dict[str, torchnet.meter.AverageValueMeter] = \ {metric_name: torchnet.meter.AverageValueMeter() for metric_name in metrics}
def main() -> None: args = get_args() config = get_bunch_config_from_json(args.config) comet_experiment = Experiment( api_key=config.comet_api_key, project_name=config.comet_project_name, workspace=config.comet_workspace, disabled=not config.use_comet_experiments, ) comet_experiment.set_name(config.experiment_name) comet_experiment.log_parameters(config) test_tweets = load_test_tweets(config.test_data_path) client = LanguageServiceClient() result = [] predictions = np.zeros(len(test_tweets), dtype=np.int32) for i, tweet in enumerate(test_tweets): start_iter_timestamp = time.time() document = types.Document( type=enums.Document.Type.PLAIN_TEXT, content=tweet, language="en" ) response = client.analyze_sentiment(document=document) response_dict = MessageToDict(response) result.append(response_dict) prediction_present = bool(response_dict["documentSentiment"]) if prediction_present: # -1, 1 predictions predictions[i] = 2 * (response.document_sentiment.score > 0) - 1 print("iteration", i, "took:", time.time() - start_iter_timestamp, "seconds") comet_experiment.log_asset_data(result, name="google_nlp_api_response.json") ids = np.arange(1, len(test_tweets) + 1).astype(np.int32) predictions_table = np.column_stack((ids, predictions)) if comet_experiment.disabled: save_path = build_save_path(config) os.makedirs(save_path) formatted_predictions_table = pd.DataFrame( predictions_table, columns=["Id", "Prediction"], dtype=np.int32, ) formatted_predictions_table.to_csv( os.path.join(save_path, "google_nlp_api_predictions.csv"), index=False ) else: comet_experiment.log_table( filename="google_nlp_api_predictions.csv", tabular_data=predictions_table, headers=["Id", "Prediction"], ) percentage_predicted = np.sum(predictions != 0) / predictions.shape[0] comet_experiment.log_metric(name="percentage predicted", value=percentage_predicted)
class CometTracker: def __init__(self, comet_params, run_params=None, prev_exp_id=None): if prev_exp_id: # previous experiment api_key = comet_params['api_key'] del comet_params[ 'api_key'] # removing this because the rest of the items need to be passed self.experiment = ExistingExperiment( api_key=api_key, previous_experiment=prev_exp_id, **comet_params) print( f'In CometTracker: ExistingExperiment initialized with id: {prev_exp_id}' ) else: # new experiment self.experiment = Experiment(**comet_params) self.experiment.log_parameters(run_params) def track_metric(self, metric, value, step): self.experiment.log_metric(metric, value, step) def add_tags(self, tags): self.experiment.add_tags(tags) print(f'In [add_tags]: Added these tags to the new experiment: {tags}') def set_name(self, name): self.experiment.set_name(name)
def fit_validate(exp_params, k, data_path, write_path, others=None, custom_tag=''): """Fit model and compute metrics on train and validation set. Intended for hyperparameter search. Only logs final metrics and scatter plot of final embedding. Args: exp_params(dict): Parameter dict. Should at least have keys model_name, dataset_name & random_state. Other keys are assumed to be model parameters. k(int): Fold identifier. data_path(str): Data directory. write_path(str): Where to write temp files. others(dict): Other things to log to Comet experiment. custom_tag(str): Custom tag for comet experiment. """ # Comet experiment exp = Experiment(parse_args=False) exp.disable_mp() custom_tag += '_validate' exp.add_tag(custom_tag) exp.log_parameters(exp_params) if others is not None: exp.log_others(others) # Parse experiment parameters model_name, dataset_name, random_state, model_params = parse_params(exp_params) # Fetch and split dataset. data_train = getattr(grae.data, dataset_name)(split='train', random_state=random_state, data_path=data_path) data_train, data_val = data_train.validation_split(random_state=FOLD_SEEDS[k]) # Model m = getattr(grae.models, model_name)(random_state=FOLD_SEEDS[k], **model_params) m.write_path = write_path m.data_val = data_val with exp.train(): m.fit(data_train) # Log plot m.comet_exp = exp m.plot(data_train, data_val, title=f'{model_name} : {dataset_name}') # Probe embedding prober = EmbeddingProber() prober.fit(model=m, dataset=data_train, mse_only=True) train_z, train_metrics = prober.score(data_train, is_train=True) # Log train metrics exp.log_metrics(train_metrics) with exp.validate(): val_z, val_metrics = prober.score(data_val) # Log train metrics exp.log_metrics(val_metrics) # Log marker to mark successful experiment exp.log_other('success', 1)
def main(logger, optimized_function, optimizer, diff_scheme, optimizer_config_file, project_name, work_space, tags, num_repetitions, n, h, use_true_grad, init_psi, p): device = torch.device('cpu') print("Using device = {}".format(device)) optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') init_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',')]).float().to(device) psi_dim = len(init_psi) optimized_function_cls = str_to_class(optimized_function) optimizer_cls = str_to_class(optimizer) diff_scheme_func = str_to_class(diff_scheme) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameter('optimizer_type', optimizer) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) logger = str_to_class(logger)(experiment) y_model = optimized_function_cls(device=device, psi_init=init_psi) if use_true_grad: ndiff = y_model else: grad_func = lambda f, x, n, h: compute_gradient_of_vector_function( f=f, x=x, n=n, h=h, scheme=diff_scheme_func) ndiff = NumericalDifferencesModel(y_model=y_model, psi_dim=psi_dim, y_dim=1, x_dim=1, n=n, h=h, num_repetitions=num_repetitions, grad_func=grad_func) max_iters = optimizer_config['max_iters'] optimizer_config['max_iters'] = 1 optimizer_config['p'] = p optimizer = optimizer_cls(oracle=ndiff, x=init_psi, **optimizer_config) for iter in range(max_iters): current_psi, status, history = optimizer.optimize() print(current_psi) # if iter % 10 == 0: logger.log_performance(y_sampler=y_model, current_psi=current_psi, n_samples=5000) torch.cuda.empty_cache() logger.log_optimizer(optimizer)
def init_callbacks(self): if (self.config.model.name == "encoder"): import keras from keras.callbacks import ModelCheckpoint, TensorBoard, ReduceLROnPlateau, EarlyStopping else: import tensorflow.keras as keras from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, ReduceLROnPlateau, EarlyStopping self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, 'best_model-%s.hdf5' % self.config.callbacks.checkpoint_monitor), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, )) self.callbacks.append( ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, min_lr=0.0001)) self.callbacks.append( EarlyStopping(monitor='val_loss', patience=10, verbose=1), ) # 在TCN中使用了tensorflow_addson中的WeightNormalization层,与tensorboard不兼容 # if (self.config.model.name != "tcn"): # self.callbacks.append( # TensorBoard( # log_dir=self.config.callbacks.tensorboard_log_dir, # write_graph=self.config.callbacks.tensorboard_write_graph, # histogram_freq=1, # ) # ) # if self.config.dataset.name == "ptbdb": # self.callbacks.append( # AdvancedLearnignRateScheduler(monitor='val_main_output_loss', patience=6, verbose=1, mode='auto', # decayRatio=0.1), # ) if ("comet_api_key" in self.config): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.disable_mp() experiment.log_parameters(self.config["trainer"]) self.callbacks.append(experiment.get_callback('keras'))
def train_hierarchy_extractor( data_path, bsz, num_workers, lr, weight_decay, warmup_updates, max_updates, accumulation_steps, validation_interval, seed, model_path, tag, device, save_metric, save_min, ): experiment = Experiment(project_name="information-retrieval", auto_output_logging=False) experiment.add_tags([tag]) parameters = { "bsz": bsz, "num_workers": num_workers, "lr": lr, "weight_decay": weight_decay, "warmup_updates": warmup_updates, "max_updates": max_updates, "validation_interval": validation_interval, "seed": seed, "model_path": model_path, "device": device, "accumulation_steps": accumulation_steps, "save_metric": save_metric, "save_min": save_min } experiment.log_parameters(parameters) model = BertHierarchyExtractor(model_path, 12, device) trainer = BertExtractorTrainer( experiment, model, data_path, model_path, bsz, num_workers, lr, weight_decay, warmup_updates, max_updates, accumulation_steps, validation_interval, save_metric, save_min, device, seed, ) try: trainer.train() except Exception as e: raise e
def main(): precision = torch.float # Parameter and Object declarations env_params = { "data path": "C:/Users/aaa2cn/Documents/nao_data/", "ip": "localhost", "port": 52232, "score type": "score" # Aggregate error in pose } env = env_factory.make_env("nao", "pose assumption", env_params) # Make a pool object model_params = { "precision": precision, "weight initialization scheme": "Sparse", "grad": False } model = model_factory.make_model("NAO FC model", model_params) # Make an algorithm object alg_params = { "target": env.target, "minimization mode": env.minimize, "minimum entropy": 0.1, "tolerance": 0.1, "max steps": 64, "memory size": 10 } alg = algorithm_factory.make_alg("local search", model, alg_params) experiment = Experiment(api_key="5xNPTUDWzZVquzn8R9oEFkUaa", project_name="nao", workspace="aromorin") experiment.set_name("Pose Assumption virtual") hyper_params = { "Algorithm": "LS", "Parameterization": 35000, "Decay Factor": 0.01, "Directions": 10, "Search Radius": 0.1 } experiment.log_parameters(hyper_params) slv_params = {"environment": env, "algorithm": alg, "logger": experiment} slv = solver_factory.make_slv("robot", slv_params) slv.solve(iterations=5000) slv.save_elite_weights(path='', name='pose_assump_virtual') # Recreate the target pose alg.eval() pred = alg.model(env.observation) angles = [p.item() for p in pred] print("These are the angles: ") print(angles) env.set_joints(angles) env.say("Is this the pose you set for me?") env.rest()
def main() -> None: args = get_args() config = get_bunch_config_from_json(args.config) comet_experiment = Experiment( api_key=config.comet_api_key, project_name=config.comet_project_name, workspace=config.comet_workspace, disabled=not config.use_comet_experiments, ) comet_experiment.set_name(config.experiment_name) comet_experiment.log_parameters(config) if config.model == "randomforest": classifier = GloveEmbeddingsClassifier( RandomForestClassifier(random_state=config.random_seed)) elif config.model == "logregression": classifier = GloveEmbeddingsClassifier( LogisticRegression(solver="saga", random_state=config.random_seed)) elif config.model == "decisiontree": classifier = GloveEmbeddingsClassifier( DecisionTreeClassifier(random_state=config.random_seed)) else: raise ValueError("chosen model not available") training_features, training_labels = classifier.generate_training_data( config) best_model, best_model_score, best_model_params = classifier.run_grid_search( config.random_seed, config.model_parameters, training_features, training_labels) comet_experiment.log_metric("mean accuracy", best_model_score) comet_experiment.log_parameters(best_model_params) test_data_features = classifier.generate_test_data_features(config) ids = np.arange(1, test_data_features.shape[0] + 1) predictions = best_model.predict(test_data_features) predictions_table = np.stack([ids, predictions], axis=-1).astype(int) if comet_experiment.disabled: save_path = build_save_path(config) os.makedirs(save_path) formatted_predictions_table = pd.DataFrame( predictions_table, columns=["Id", "Prediction"], dtype=np.int32, ) formatted_predictions_table.to_csv(os.path.join( save_path, "test_predictions.csv"), index=False) else: comet_experiment.log_table( filename="test_predictions.csv", tabular_data=predictions_table, headers=["Id", "Prediction"], )
def setup_comet_ml_experiment(api_key, project_name, experiment_name, parameters, tags): """ Function for setting up comet ml experiment """ experiment = Experiment(api_key=api_key, project_name=project_name, auto_metric_logging=False) experiment.set_name(experiment_name) experiment.log_parameters(parameters) experiment.add_tags(tags) return experiment
def main(logger, optimizer, optimized_function, optimizer_config_file, model_config_file, project_name, work_space, tags, reuse_optimizer, init_psi, epochs): model_config = getattr(__import__(model_config_file), 'model_config') optimizer_config = getattr(__import__(optimizer_config_file), 'optimizer_config') current_psi = torch.tensor([float(x.strip()) for x in init_psi.split(',') ]).float().to(device) psi_dim = len(init_psi) print(psi_dim, current_psi) optimized_function_cls = str_to_class(optimized_function) optimizer_cls = str_to_class(optimizer) experiment = Experiment(project_name=project_name, workspace=work_space) experiment.add_tags([x.strip() for x in tags.split(',')]) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.items() }) experiment.log_parameters({ "optimizer_{}".format(key): value for key, value in optimizer_config.get('line_search_options', {}).items() }) logger = str_to_class(logger)(experiment) y_model = optimized_function_cls(device=device, psi_init=current_psi) model = VoidModel(y_model=y_model, **model_config) optimizer = optimizer_cls(oracle=model, x=current_psi, **optimizer_config) for epoch in range(epochs): if reuse_optimizer: optimizer.update(oracle=model, x=current_psi) else: # find new psi optimizer = optimizer_cls(oracle=model, x=current_psi, **optimizer_config) current_psi, status, history = optimizer.optimize() try: logger.log_optimizer(optimizer) logger.log_grads(model, y_sampler=y_model, current_psi=current_psi, num_repetitions=5000) logger.log_performance(y_sampler=y_model, current_psi=current_psi, n_samples=5000) except Exception as e: print(e) raise
def main(cmd=None, stdout=True): """Run finetuning experiment for fixed seed.""" # Initialize system args = get_args(cmd) assert torch.cuda.is_available() torch.cuda.set_device(args.device) # Initialize logging model_id = ("Finetune {}, seed size {}, epochs {}, labels {}, " "batch size {}, lr {}").format(args.model, args.seed_size, args.epochs, args.label_budget, args.batch_size, args.lr) logging.basicConfig(filename="{}/{}.txt".format(args.dout, model_id), format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO) if stdout: logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logger = Experiment(comet_ml_key, project_name="ActiveDialogue") logger.set_name(model_id) logger.log_parameters(vars(args)) # Select model and environment if args.model == "glad": model_arch = GLAD elif args.model == "gce": model_arch = GCE env = DSTEnv(load_dataset, model_arch, args) # Load seed if need-be if not env.load('seed'): raise ValueError("No loaded seed.") # Initialize evaluation best_metrics = env.metrics(True) for k, v in best_metrics.items(): logger.log_metric(k, v, step=0) logging.info("Initial metrics: %s", best_metrics) # Finetune env.label_all() for epoch in range(1, args.epochs + 1): logging.info('Starting fit epoch %d.', epoch) env.fit() metrics = env.metrics(True) logging.info("Epoch metrics: %s", metrics) for k, v in metrics.items(): logger.log_metric(k, v, step=epoch) if best_metrics is None or metrics[args.stop] > best_metrics[ args.stop]: logging.info("Saving best!") best_metrics = metrics
def main(): hyper_params = { "learning_rate": 1, "style_weight": 10000, "content_weight": 1, "n_steps": 300 } experiment = Experiment(api_key="a604AfX0S9Bmt6HdpMHxg9MCI", project_name="style-transfer", workspace="polmonroig") experiment.log_parameters(hyper_params) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if torch.cuda.is_available(): print("Currently using cuda device.") # define file paths and directories content_images_dir = "content_images/" style_images_dir = "style_images/" output_images_dir = "output_images/" content_image_name = "content_01.png" style_image_name = "style_01.jpg" output_image_path = join( output_images_dir, content_image_name.split('.')[0] + "_" + style_image_name.split('.')[0] + ".jpg") content_image_path = join(content_images_dir, content_image_name) style_image_path = join(style_images_dir, style_image_name) # define image file manager max_shape = (800, 800) fileManager = FileManager(content_image_path, style_image_path, device, max_shape) # read images content_image, style_image = fileManager.read_images() input_image = content_image.clone() model = ArtNet(device=device) output_image = model.train(hyper_params['content_weight'], hyper_params['style_weight'], hyper_params['n_steps'], content_image, style_image, input_image, hyper_params['learning_rate'], experiment) fileManager.save_image(output_image, output_image_path) experiment.log_image( output_image_path, content_image_name.split('.')[0] + "_" + style_image_name.split('.')[0])
def train_simultaneous(hyper_params, teacher, student, sf_teacher, sf_student, trainloader, valloader, args): if args.api_key: project_name = 'simultaneous-' + hyper_params[ 'dataset'] + '-' + hyper_params['model'] experiment = Experiment(api_key=args.api_key, project_name=project_name, workspace=args.workspace) experiment.log_parameters(hyper_params) optimizer = torch.optim.Adam(student.parameters(), lr=hyper_params['learning_rate']) scheduler = torch.optim.lr_scheduler.OneCycleLR( optimizer, max_lr=1e-2, steps_per_epoch=len(trainloader), epochs=hyper_params['num_epochs']) if hyper_params['dataset'] == 'camvid': criterion = nn.CrossEntropyLoss(ignore_index=11) else: criterion = nn.CrossEntropyLoss(ignore_index=250) hyper_params['num_classes'] = 19 criterion2 = nn.MSELoss() savename = get_savename(hyper_params, dataset=args.dataset, mode='simultaneous', p=args.percentage) highest_iou = 0 for epoch in range(hyper_params['num_epochs']): _, _, train_loss, val_loss, avg_iou, avg_px_acc, avg_dice_coeff = train_simult( model=student, teacher=teacher, sf_teacher=sf_teacher, sf_student=sf_student, train_loader=trainloader, val_loader=valloader, num_classes=hyper_params['num_classes'], loss_function=criterion, loss_function2=criterion2, optimiser=optimizer, scheduler=scheduler, epoch=epoch, num_epochs=hyper_params['num_epochs'], savename=savename, highest_iou=highest_iou, args=args) if args.api_key: experiment.log_metric('train_loss', train_loss) experiment.log_metric('val_loss', val_loss) experiment.log_metric('avg_iou', avg_iou) experiment.log_metric('avg_pixel_acc', avg_px_acc) experiment.log_metric('avg_dice_coeff', avg_dice_coeff)
def multi_train(configs_path="./configs.json"): # configs_path = "/home/shared/Layer-Wise-Learning-Trends-PyTorch/configs.json" print(f"Configs Path: {configs_path}") configs = SimpleNamespace(**json.load(open(configs_path))) print(configs) experiment = Experiment( api_key="ZgD8zJEiZErhwIzPMfZpitMjq", project_name="imagenet-rwc", workspace="ayushm-agrawal", ) experiment.log_parameters(configs) average_rmae_dict, average_train_acc, average_test_acc = { }, np.zeros(configs.epochs), np.zeros(configs.epochs) og_config_exp_name = configs.exp_name for _, seed in enumerate(configs.seed_list): configs.exp_name = f"{seed}_{og_config_exp_name}" print(f"Training: {configs.exp_name}") configs.seed = seed configs.experiment = experiment rmae_dict, train_acc_arr, test_acc_arr = run_experiment( configs.epochs, configs.model_name, "untrained", configs) for layer in rmae_dict: if layer not in average_rmae_dict: average_rmae_dict[layer] = np.array(rmae_dict[layer]) else: average_rmae_dict[layer] += np.array(rmae_dict[layer]) average_train_acc += train_acc_arr average_test_acc += test_acc_arr for layer in average_rmae_dict: average_rmae_dict[layer] /= len(configs.seed_list) average_train_acc /= len(configs.seed_list) average_test_acc /= len(configs.seed_list) save_path = join(configs.arr_save_path, og_config_exp_name) print(f"Saving Arrays at {save_path}") np.save(save_path + "avg_train_acc.npy", average_train_acc) np.save(save_path + "avg_test_acc.npy", average_test_acc) np.save(save_path + "avg_rmae_dict.npy", average_rmae_dict) print("Done!")
def main(cfg, comet=False): cfg = Config(cfg) print('start') # comet-ml setting if comet: experiment = Experiment(api_key=cfg.api_key, project_name=cfg.project_name, workspace=cfg.workspace) experiment.log_parameters(cfg) else: experiment = None # device and dataset setting #device = (torch.device(f'cuda:{cfg.gpu_id}') # if torch.cuda.is_available() and cfg.gpu_id >= 0 # else torch.device('cpu')) device = torch.device('cuda:0') print(device) dataset = MCDataset(cfg.root, cfg.dataset_name) print('load done') data = dataset[0].to(device) # add some params to config cfg.num_nodes = dataset.num_nodes cfg.num_relations = dataset.num_relations cfg.num_users = int(data.num_users) # set and init model model = GAE(cfg, random_init).to(device) model.cuda() model.apply(init_xavier) # optimizer optimizer = torch.optim.Adam( model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay, ) # train trainer = Trainer( model, dataset, data, calc_rmse, optimizer, experiment, ) trainer.training(cfg.epochs)