def run(): if 'COMET_KEY' not in os.environ: print('Please set the COMET_KEY environment variable') print('e.g. by running with: `COMET_KEY="your-key" python example.py`') return api_key = os.environ['COMET_KEY'] # Create an experiment experiment = Experiment(api_key=api_key, project_name="tabular_q", workspace="irl-benchmark") metrics_logger = MetricsLogger() def metric_listener(reward): experiment.log_metric('reward', reward) metrics_logger.log_metric('reward', reward) duration = 5 hyper_params = {'gamma': 0.95, 'alpha_start': 0.8} # Report hparams experiment.log_multiple_params(hyper_params) env = gym.make('FrozenLake-v0') agent = TabularQ(env, **hyper_params) rewards = agent.train(duration, metric_listener) experiment.log_metric('MA(100) reward', np.mean(rewards[-100:])) metrics_logger.save('data/example_tabular_q_metrics.json')
def run(**hyper_params): if 'COMET_KEY' not in os.environ: print('Please set the COMET_KEY environment variable') print('e.g. by running with: `COMET_KEY="your-key" python example.py`') sys.exit() api_key = os.environ['COMET_KEY'] # Create an experiment experiment = Experiment( api_key=api_key, project_name="tabular_q", workspace="irl-benchmark") def metric_listener(reward): experiment.log_metric('reward', reward) duration = 5 # Report hparams experiment.log_multiple_params(hyper_params) env = gym.make('FrozenLake-v0') agent = TabularQ(env, **hyper_params) rewards = agent.train(duration, metric_listener) ma_100 = np.mean(rewards[-100:]) experiment.log_metric('MA(100) reward', ma_100) print('Params:', hyper_params, 'got ma_100 reward', ma_100) return rewards
def main(): config = load_config() if config['random_seed']: random.seed(config['random_seed']) torch.manual_seed(config['random_seed']) disable_cometml = not config['use_cometml'] experiment = Experiment(api_key=config['comet_key'], project_name=config['project_name'], disabled=disable_cometml) model_config = config['model'] experiment.log_multiple_params(model_config) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') train_dataset = StockDataset(config, device=device, mode='train') train_data_loader = DataLoader(train_dataset, batch_size=config['model']['batch_size'], shuffle=True) dev_dataset = StockDataset(config, device=device, mode='dev') dev_data_loader = DataLoader(dev_dataset, batch_size=config['model']['batch_size'], shuffle=False) model = Model(input_size=model_config['input_size'], hidden_size=model_config['hidden_size'], output_size=model_config['output_size'], num_layers=model_config['num_layers'], dropout=model_config['dropout'], device=device) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=model_config['learning_rate']) train_and_evaluate(model=model, criterion=criterion, optimizer=optimizer, train_data_loader=train_data_loader, dev_data_loader=dev_data_loader, epochs_count=model_config['epochs_count'], experiment=experiment)
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config['exp_name']), monitor=self.config['checkpoint_monitor'], mode=self.config['checkpoint_mode'], save_best_only=self.config['checkpoint_save_best_only'], save_weights_only=self.config['checkpoint_save_weights_only'], verbose=self.config['checkpoint_verbose'], )) self.callbacks.append( TensorBoard( log_dir=self.tensorboard_log_dir, write_graph=self.config['tensorboard_write_graph'], histogram_freq=0, # don't compute histograms write_images= False # don't write model weights to visualize as image in TensorBoard )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config['comet_api_key'], project_name=self.config['exp_name']) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def train_autofeature_model(data_path, embedding_dimension, batch_size): training_data = load_data(data_path) X = training_data.drop(labels=['SK_ID_CURR'], axis=1) experiment = Experiment( api_key=API_KEY, project_name="home-credit") experiment.set_name( 'home-credit-autofeature-selection') model = build_model(X.shape[1], int(embedding_dimension)) model.summary() model.compile(optimizer='adam', loss='mean_squared_logarithmic_error') model.fit(X, X, epochs=5, batch_size=int(batch_size)) experiment.log_multiple_params( {"embedding_dimension": embedding_dimension, "batch_size": batch_size}) model.save( 'home-credit-encoder-{}-{}.hdf5'.format( embedding_dimension, batch_size))
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join(self.config.checkpoint_dir, '%s-{epoch:03d}-{val_nme:.5f}.hdf5' % self.config.exp_name), monitor=self.config.checkpoint_monitor, mode=self.config.checkpoint_mode, save_best_only=self.config.checkpoint_save_best_only, save_weights_only=self.config.checkpoint_save_weights_only, verbose=self.config.checkpoint_verbose, ) ) self.callbacks.append( TensorBoard( log_dir=self.config.tensorboard_log_dir, write_graph=self.config.tensorboard_write_graph, ) ) # self.callbacks.append( # LearningRateScheduler(self.lr_scheduler) # ) if hasattr(self.config,"comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def train(path): name = os.path.splitext(os.path.basename(path))[0] print('Processing: ', name) features = pd.read_csv(path, index_col=None) selected_features_names = [name for name, desc in selected_features] features = features[selected_features_names] split_idx = 1200 features = features.drop(['sound.files'], axis=1) noise_only_df, df = features.iloc[:split_idx], features.iloc[split_idx:] y = df.pop('petrel') X = df.values y_noise = noise_only_df.pop('petrel') X_noise = noise_only_df.values X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.25, random_state=42, stratify=y) hyperparams = { 'n_estimators': [100, 300, 500, 1000], 'learning_rate': [0.1], 'gamma': [0.0, 0.5], 'max_depth': [2, 3, 4], 'min_child_weight': [1, 2], 'subsample': [1.0, 0.8], 'reg_alpha': [0.0, 0.1], 'reg_lambda': [1, 2, 3] } # # hyperparams = { # 'n_estimators': [100], # 'learning_rate': [0.1], # 'gamma': [0.0], # 'max_depth': [2], # 'min_child_weight': [1], # 'subsample': [1.0], # 'reg_alpha': [0.0], # 'reg_lambda': [1] # } clf = model_selection.GridSearchCV(estimator=xg.XGBClassifier(objective='binary:logistic', n_jobs=-1), param_grid=hyperparams, cv=4) fit_params = clf.fit(X_train, y_train) estimator = fit_params.best_estimator_ joblib.dump(estimator, name + '_model.pkl') test_pred = estimator.predict(X_test) metrics = calculate_metrics(test_pred, y_test) noise_pred = estimator.predict(X_noise) noise_detection_accuracy = accuracy_score(y_noise, noise_pred) experiment = Experiment(api_key="4PdGdUZmGf6P8QsMa5F2zB4Ui", project_name="storm petrels", workspace="tracewsl") experiment.set_name(name) experiment.log_parameter('name', name) experiment.log_multiple_params(fit_params.best_params_) experiment.log_multiple_metrics(metrics) experiment.log_metric('Noise detection accuracy', noise_detection_accuracy) experiment.log_figure('Confusion matrix', get_confusion_matrix_figure(test_pred, y_test)) experiment.log_figure('Feature importnace', get_feature_importance_figure(estimator, list(df.columns.values)))
class Logger: def __init__(self, sess, config): self.sess = sess self.config = config self.summary_placeholders = {} self.summary_ops = {} self.train_summary_writer = tf.summary.FileWriter(os.path.join(self.config.summary_dir, "train"), self.sess.graph) self.test_summary_writer = tf.summary.FileWriter( os.path.join(self.config.summary_dir, "test")) if "comet_api_key" in config: from comet_ml import Experiment self.experiment = Experiment( api_key=config['comet_api_key'], project_name=config['exp_name']) self.experiment.disable_mp() self.experiment.log_multiple_params(config) # it can summarize scalars and images. def summarize(self, step, summarizer="train", scope="", summaries_dict=None): """ :param step: the step of the summary :param summarizer: use the train summary writer or the test one :param scope: variable scope :param summaries_dict: the dict of the summaries values (tag,value) :return: """ summary_writer = self.train_summary_writer if summarizer == "train" else self.test_summary_writer with tf.variable_scope(scope): if summaries_dict is not None: summary_list = [] for tag, value in summaries_dict.items(): if tag not in self.summary_ops: if len(value.shape) <= 1: self.summary_placeholders[tag] = tf.placeholder( 'float32', value.shape, name=tag) else: self.summary_placeholders[tag] = tf.placeholder('float32', [None] + list(value.shape[1:]), name=tag) if len(value.shape) <= 1: self.summary_ops[tag] = tf.summary.scalar( tag, self.summary_placeholders[tag]) else: self.summary_ops[tag] = tf.summary.image( tag, self.summary_placeholders[tag]) summary_list.append(self.sess.run(self.summary_ops[tag], { self.summary_placeholders[tag]: value})) for summary in summary_list: summary_writer.add_summary(summary, step) if hasattr(self, 'experiment') and self.experiment is not None: self.experiment.log_multiple_metrics( summaries_dict, step=step) summary_writer.flush()
def train(): x_lines = [ *toolz.take(LIMIT, open('data/x.txt').read().lower().split('\n')) ] y_lines = [ *toolz.take(LIMIT, open('data/y.txt').read().lower().split('\n')) ] encoder = encoder_for_lines(S2S_PARAMS, x_lines + y_lines) try: start_idx = encoder.word_vocab[S2S_PARAMS.start_token] pad_idx = encoder.word_vocab[PAD_TOKEN] except AttributeError: start_idx = int(encoder.vocabulary_[S2S_PARAMS.start_token]) pad_idx = encoder.vocabulary_[PAD_TOKEN] reverse_enc = {idx: word for word, idx in encoder.vocabulary_.items()} model = build_model(S2S_PARAMS, start_idx, pad_idx) x = encode_data(encoder, x_lines, is_input=True) y = encode_data(encoder, y_lines, is_input=False) print(x.shape, y.shape) x = x[:S2S_PARAMS.batch_size * int(len(x) / S2S_PARAMS.batch_size)] y = y[:S2S_PARAMS.batch_size * int(len(y) / S2S_PARAMS.batch_size)] test_x = x[:S2S_PARAMS.batch_size] losses = [] if USE_COMET: experiment = Experiment(api_key="DQqhNiimkjP0gK6c8iGz9orzL", log_code=True) experiment.log_multiple_params(S2S_PARAMS._asdict()) for idx in range(1000): print("Shuffling data...") random_idx = random.sample([*range(len(x))], len(x)) x = x[random_idx] y = y[random_idx] print("Training in epoch " + str(idx)) losses.append(model.train_epoch(x, y, experiment=experiment)) experiment.log_epoch_end(idx) print('Loss history: {}'.format(', '.join( ['{:.4f}'.format(loss) for loss in losses]))) test_y = model.predict(test_x) for i in range(min([3, S2S_PARAMS.batch_size])): print('> ' + ' '.join( reverse_enc.get(idx, '<unk/>') for idx in list(test_y[i]))) else: for idx in range(1000): print("Training in epoch " + str(idx)) model.train_epoch(x, y)
def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('--MIN_LENGTH', type=int, default=1, help='Min Length of sequence (Input side)') parser.add_argument('--MAX_LENGTH', type=int, default=40, help='Max Length of sequence (Input side)') parser.add_argument('--MIN_LENGTH_TARGET', type=int, default=1, help='Min Length of sequence (Output side)') parser.add_argument('--MAX_LENGTH_TARGET', type=int, default=40, help='Max Length of sequence (Output side)') parser.add_argument('--lang1', type=str, default="en", help='Input Language') parser.add_argument('--lang2', type=str, default="fr", help='Target Language') parser.add_argument('--USE_CUDA', action='store_true', help='IF USE CUDA (Default == False)') parser.add_argument('--attention', type=str, default='Bahdanau', help='attention type: either Bahdanau or Luong') # parser.add_argument('--teacher_forcing_ratio', type=float, default=1, help='Teacher forcing ratio for encoder') parser.add_argument('--hidden_size', type=int, default=1024, help='Size of hidden layer') parser.add_argument('--n_epochs', type=int, default=50000, help='Number of single iterations through the data') parser.add_argument('--learning_rate', type=float, default=0.0001, help='Learning rate (for both, encoder and decoder)') parser.add_argument('--n_layers', type=int, default=2, help='Number of layers (for both, encoder and decoder)') parser.add_argument('--dropout', type=float, default=0.1, help='Dropout (%) in the decoder') parser.add_argument('--model_type', type=str, default="seq2seq", help='Model type (and ending of files)') parser.add_argument('--main_data_dir', type=str, default= "/scratch/eff254/NLP/Data/Model_ready", help='Directory where data is saved (in folders tain/dev/test)') parser.add_argument('--out_dir', type=str, default="./checkpoints", help="Directory to save the models state dict (No default)") parser.add_argument('--eval_dir', type=str, default="/scratch/eff254/NLP/Evaluation", help="Directory to save predictions - MUST CONTAIN PEARL SCRIPT") parser.add_argument('--optimizer', type=str, default="Adam", help="Optimizer (Adam vs SGD). Default: Adam") parser.add_argument('--kmax', type=int, default=10, help="Beam search Topk to search") parser.add_argument('--clip', type=int, default=1, help="Clipping the gradients") parser.add_argument('--batch_size', type=int, default=128, help="Size of a batch") parser.add_argument('--min_count_trim_output', type=int, default=2, help="trim infrequent output words") parser.add_argument('--min_count_trim_input', type=int, default=2, help="trim infrequent input words") parser.add_argument('--save_every', type=int, default=100, help='Checkpoint model after number of iters') parser.add_argument('--print_every', type=int, default=10, help='Print training loss after number of iters') parser.add_argument('--eval_every', type=int, default=100, help='Evaluate translation on one dev pair after number of iters') parser.add_argument('--bleu_every', type=int, default=200, help='Get bleu score number of iters') parser.add_argument('--scheduled_sampling_k', type=int, default=3000, help='scheduled sampling parameter for teacher forcing, \ based on inverse sigmoid decay') parser.add_argument('--experiment', type=str, default="MICA", help='experiment name') opt = parser.parse_args() print(opt) if opt.experiment is None: opt.experiment = 'MICA_experiment' target_char = (opt.model_type == 'bpe2char') if target_char: opt.MAX_LENGTH_TARGET = 200 ######## Comet ML ######## #experiment = comet_mirror("Experiment2") experiment = Experiment(api_key="00Z9vIf4wOLZ0yrqzdwHqttv4", log_code=True) hyper_params = vars(opt) experiment.log_multiple_params(hyper_params) # flag for character encoding return opt, target_char, experiment
def init_comet(params, trainer): if params['comet_key'] is not None: from comet_ml import Experiment from trainer.plugins import CometPlugin experiment = Experiment(api_key=params['comet_key'], log_code=False) hyperparams = { name: param_to_string(params[name]) for name in tag_params } experiment.log_multiple_params(hyperparams) trainer.register_plugin( CometPlugin(experiment, [('training_loss', 'epoch_mean'), 'validation_loss', 'test_loss']))
def init_comet(params, trainer): if params['comet_key']: from comet_ml import Experiment experiment = Experiment(api_key=params['comet_key'], project_name=params['comet_project_name'], log_code=False) hyperparams = {name: str(params[name]) for name in params} experiment.log_multiple_params(hyperparams) trainer.register_plugin( CometPlugin(experiment, [ 'G_loss.epoch_mean', 'D_loss.epoch_mean', 'D_real.epoch_mean', 'D_fake.epoch_mean', 'sec.kimg', 'sec.tick', 'kimg_stat' ] + (['depth', 'alpha'] if params['progressive_growing'] else []))) else: print('Comet_ml logging disabled.')
def train(hyper_params): mnist = get_data() # Get graph definition, tensors and ops train_step, cross_entropy, accuracy, x, y, y_ = build_model_graph( hyper_params) # log parameters to Comet.ml import os # Setting the API key (saved as environment variable) exp = Experiment( #api_key="YOUR API KEY", # or api_key=os.environ.get("COMET_API_KEY"), project_name='comet-examples') exp.log_multiple_params(hyper_params) exp.log_dataset_hash(mnist) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) exp.set_model_graph(sess.graph) for i in range(hyper_params["steps"]): batch = mnist.train.next_batch(hyper_params["batch_size"]) exp.set_step(i) # Compute train accuracy every 10 steps if i % 10 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch[0], y_: batch[1] }) print('step %d, training accuracy %g' % (i, train_accuracy)) exp.log_metric("acc", train_accuracy) # Update weights (back propagation) loss = train_step.run(feed_dict={x: batch[0], y_: batch[1]}) exp.log_metric("loss", loss) ### Finished Training ### # Compute test accuracy acc = accuracy.eval(feed_dict={ x: mnist.test.images, y_: mnist.test.labels }) print('test accuracy %g' % acc)
class Logger(object): def __init__(self, dataset_name, model_name): self.model_name = model_name self.project_name = "%s-%s" % (dataset_name, self.model_name) self.logdir = os.path.join(hp.logdir, self.project_name) self.writer = SummaryWriter(log_dir=self.logdir) self.experiment = None # Experiment(api_key="luY5eUQDsBynS168WxJiRPJmJ", project_name=self.project_name, log_code=False) if hp.comet_ml_api_key is not None: self.experiment = Experiment(api_key=hp.comet_ml_api_key, project_name=self.project_name, log_code=False) self.experiment.log_multiple_params( dict((name, getattr(hp, name)) for name in dir(hp) if not name.startswith('__'))) def log_step(self, phase, step, loss_dict, image_dict): if phase == 'train': if step % 50 == 0: if self.experiment is not None: with self.experiment.train(): self.experiment.log_multiple_metrics(loss_dict, step=step) # self.writer.add_scalar('lr', get_lr(), step) # self.writer.add_scalar('%s-step/loss' % phase, loss, step) for key in sorted(loss_dict): self.writer.add_scalar('%s-step/%s' % (phase, key), loss_dict[key], step) if step % 1000 == 0: for key in sorted(image_dict): self.writer.add_image('%s/%s' % (self.model_name, key), image_dict[key], step) def log_epoch(self, phase, step, loss_dict): for key in sorted(loss_dict): self.writer.add_scalar('%s/%s' % (phase, key), loss_dict[key], step) if phase == 'valid': if self.experiment is not None: with self.experiment.validate(): self.experiment.log_multiple_metrics(loss_dict, step=step)
def train(self, **kwargs): self._convert_annotation_library_to_tfrecords() luminoth_config, hyperparams = self._generate_train_config(**kwargs) experiment = Experiment( api_key=config["credentials"]["comet_ml"]["api_key"], project_name=config["credentials"]["comet_ml"]["project_name"], log_code=False, log_graph=False, auto_param_logging=False, auto_metric_logging=False, auto_output_logging=None, log_env_details=False, log_git_metadata=False ) experiment.log_multiple_params(hyperparams) luminoth_train(luminoth_config, environment="local", experiment=experiment)
def init_comet(params, trainer): if params['comet_key']: from comet_ml import Experiment experiment = Experiment(api_key=params['comet_key'], project_name=params['comet_project_name'], log_code=False) hyperparams = { name: str(params[name]) for name in params } experiment.log_multiple_params(hyperparams) trainer.register_plugin(CometPlugin( experiment, [ 'G_loss.epoch_mean', 'D_loss.epoch_mean', 'D_real.epoch_mean', 'D_fake.epoch_mean', 'sec.kimg', 'sec.tick', 'kimg_stat' ] + (['depth', 'alpha'] if params['progressive_growing'] else []) )) else: print('Comet_ml logging disabled.')
def train(hyper_params): mnist = get_data() # Get graph definition, tensors and ops train_step, cross_entropy, accuracy, x, y, y_ = build_model_graph( hyper_params) experiment = Experiment(api_key="XXXX", project_name='my project') experiment.log_multiple_params(hyper_params) experiment.log_dataset_hash(mnist) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) experiment.set_model_graph(sess.graph) for i in range(hyper_params["steps"]): batch = mnist.train.next_batch(hyper_params["batch_size"]) experiment.set_step(i) # Compute train accuracy every 10 steps if i % 10 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch[0], y_: batch[1] }) print('step %d, training accuracy %g' % (i, train_accuracy)) experiment.log_metric("acc", train_accuracy) # Update weights (back propagation) loss = train_step.run(feed_dict={x: batch[0], y_: batch[1]}) experiment.log_metric("loss", loss) ### Finished Training ### # Compute test accuracy acc = accuracy.eval(feed_dict={ x: mnist.test.images, y_: mnist.test.labels }) print('test accuracy %g' % acc)
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) # if the config has the debug flag on, turn on tfdbg (TODO: make it work) if hasattr(self.config, "debug"): if (self.config.debug == True): import keras.backend from tensorflow.python import debug as tf_debug print("#=========== DEBUG MODE ===========#") sess = keras.backend.get_session() sess = tf_debug.LocalCLIDebugWrapperSession(sess) keras.backend.set_session(sess) # if the config file has a comet_ml key, log on comet if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment # PUT the import in main experiment = Experiment(api_key=self.config.exp.comet_api_key, project_name=self.config.exp.name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def run_lightgbm(train_df, validation_df): train_data = lgb.Dataset(data=train_df.drop(columns=['TARGET']), label=train_df['TARGET']) validation_data = lgb.Dataset(data=validation_df.drop(columns=['TARGET']), label=validation_df['TARGET']) num_round = 10 params = """ num_leaves integer [31, 51] [31] num_trees integer [50, 100] [50] """ optimizer = Optimizer(API_KEY) optimizer.set_params(params) while True: suggestion = optimizer.get_suggestion() experiment = Experiment(api_key=API_KEY, project_name='home-credit') experiment.set_name('lightgbm') _param = { 'num_leaves': suggestion['num_leaves'], 'num_trees': suggestion['num_trees'], 'objective': 'binary', 'metric': 'auc' } experiment.log_multiple_params(_param) experiment.log_dataset_hash( pd.concat([train_df, validation_df], axis=0)) bst = lgb.train(_param, train_data, num_round, valid_sets=[validation_data]) y_pred = bst.predict(validation_df.drop(columns=['TARGET'])) auc_score = roc_auc_score(validation_df['TARGET'], y_pred) experiment.log_metric(name='auc_score', value=auc_score) suggestion.report_score("auc_score", auc_score)
def main(_): print("Model Architecture: {}".format(FLAGS.model_architecture)) # Adjust some parameters if FLAGS.debug: FLAGS.small_label_set = False print("RUNNING IN DEBUG MODE") FLAGS.num_classes = utils.get_num_classes(FLAGS) X_train, y_train = data_utils.load_dataset_tf(FLAGS, mode="train") X_val, y_val = data_utils.load_dataset_tf(FLAGS, mode="val") # comet_ml experiment logging (https://www.comet.ml/) experiment = Experiment(api_key="J55UNlgtffTDmziKUlszSMW2w", log_code=False) experiment.log_multiple_params(utils.gather_params(FLAGS)) experiment.set_num_of_epocs(FLAGS.epochs) experiment.log_dataset_hash(X_train) tf.logging.set_verbosity(tf.logging.INFO) # Start a new, DEFAULT TensorFlow session. sess = tf.InteractiveSession() utils.set_seeds() # Get deterministic behavior? model = models.create_model(FLAGS) fw = framework.Framework(sess, model, experiment, FLAGS) num_params = int(utils.get_number_of_params()) model_size = num_params * 4 print("\nNumber of trainable parameters: {}".format(num_params)) print("Model is ~ {} bytes out of max 5000000 bytes\n".format(model_size)) experiment.log_parameter("num_params", num_params) experiment.log_parameter("approx_model_size", model_size) fw.optimize(X_train, y_train, X_val, y_val)
def train(args): # So I don't frigging forget what caused working models save_args(args) if args["use_tf_debug"]: hooks = [tf_debug.LocalCLIDebugHook()] else: hooks = [] if args["use_comet"]: # Add the following code anywhere in your machine learning file experiment = Experiment(api_key="bRptcjkrwOuba29GcyiNaGDbj", project_name="macgraph", workspace="davidhughhenrymack") experiment.log_multiple_params(args) if len(args["tag"]) > 0: experiment.add_tags(args["tag"]) train_size = sum( 1 for _ in tf.python_io.tf_record_iterator(args["train_input_path"])) logger.info(f"Training on {train_size} records") # ---------------------------------------------------------------------------------- estimator = get_estimator(args) train_spec = tf.estimator.TrainSpec( input_fn=gen_input_fn(args, "train"), max_steps=args["train_max_steps"] * 1000 if args["train_max_steps"] is not None else None, hooks=hooks) eval_spec = tf.estimator.EvalSpec(input_fn=gen_input_fn(args, "eval"), throttle_secs=args["eval_every"]) tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
class CometMLLogger(Logger): def __init__(self, logger_kwargs=None): super().__init__(logger_kwargs=logger_kwargs) self.experiment = Experiment(api_key=self.config["api_key"], project_name=self.config["project_name"], log_code=False, log_graph=False, auto_param_logging=False, auto_metric_logging=False, auto_output_logging=None, log_env_details=False, log_git_metadata=False) if "reward_func" in self.config: self.experiment.set_code( inspect.getsource(self.config["reward_func"])) def log_hyperparams(self, hyperparams, step=0): self.experiment.log_multiple_params(hyperparams, step=step) def log_metric(self, key, value, step=0): self.experiment.log_metric(key, value, step=step)
'VOCAB_SIZE': VOCAB_SIZE, 'BATCH_SIZE': BATCH_SIZE, 'EMBED_SIZE': EMBED_SIZE, 'SKIP_WINDOW': SKIP_WINDOW, 'NUM_SAMPLED': NUM_SAMPLED, 'LEARNING_RATE': LEARNING_RATE, 'NUM_TRAIN_STEPS': NUM_TRAIN_STEPS, 'VISUAL_FLD': VISUAL_FLD, 'SKIP_STEP': SKIP_STEP, 'DOWNLOAD_URL': DOWNLOAD_URL, 'EXPECTED_BYTES': EXPECTED_BYTES, 'NUM_VISUALIZE': NUM_VISUALIZE } exp = Experiment(api_key="YOUR-API-KEY", project_name='word2vec') exp.log_multiple_params(params) def word2vec(dataset): """ Build the graph for word2vec model and train it """ # Step 1: get input, output from the dataset with tf.name_scope('data'): iterator = dataset.make_initializable_iterator() center_words, target_words = iterator.get_next() """ Step 2 + 3: define weights and embedding lookup. In word2vec, it's actually the weights that we care about """ with tf.name_scope('embed'): embed_matrix = tf.get_variable( 'embed_matrix', shape=[VOCAB_SIZE, EMBED_SIZE],
padding='same', activation=params['activation'])) model.add(Dropout(params['dropout'])) model.add(Flatten()) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=params['optimizer'], metrics=['accuracy']) #print model.summary() to preserve automatically in `Output` tab print(model.summary()) params.update({'total_number_of_parameters': model.count_params()}) #will log metrics with the prefix 'train_' with experiment.train(): model.fit(X_train, y_train, epochs=params['epochs'], batch_size=params['batch_size'], verbose=1, validation_data=(X_test, y_test)) #will log metrics with the prefix 'test_' with experiment.test(): loss, accuracy = model.evaluate(X_test, y_test) metrics = {'loss': loss, 'accuracy': accuracy} experiment.log_multiple_metrics(metrics) experiment.log_multiple_params(params) experiment.log_dataset_hash(X_train) #creates and logs a hash of your data
def main(): args = get_args() hyperparams = vars(args) if not args.no_comet: experiment = Experiment(api_key="5yzCYxgDmFnt1fhJWTRQIkETT", log_code=True) experiment.log_multiple_params(hyperparams) text_field = data.Field(tokenize=custom_tokenizer, fix_length=args.sentence_len, unk_token='<**UNK**>') label_field = data.Field(sequential=False, unk_token=None) pair_field = data.RawField() if args.dataset == 'multinli': print('Loading MultiNLI Dataset') train = get_dataset(text_field, label_field, pair_field, 'train') val = get_dataset(text_field, label_field, pair_field, args.val_set) elif args.dataset == 'snli': print('Loading SNLI Dataset') train, val, test = datasets.SNLI.splits(text_field, label_field) del test else: raise Exception('Incorrect Dataset Specified') text_field.build_vocab(train, max_size=args.max_vocab_size) label_field.build_vocab(train, val) if args.word_vectors: text_field.vocab.load_vectors(args.word_vectors) device = -1 if args.cuda: device = None print('Generating Iterators') train_iter, val_iter = data.BucketIterator.splits( (train, val), batch_size=args.batch_size, shuffle=True, sort_key=sort_key, device=device) train_iter.repeat = False args.n_embed = len(text_field.vocab) args.d_out = len(label_field.vocab) args.n_cells = args.n_layers if args.bidir: args.n_cells *= 2 print(args) if args.load_model: model = torch.load(args.load_model) else: model = MODELS[args.model_type](args) print('Loading Word Embeddings') model.embed.weight.data = text_field.vocab.vectors criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr) if args.cuda: model = model.cuda() criterion = criterion.cuda() print(model) print('Training Model') best_val_acc = 0.0 val_acc_history = [] for epoch in range(1, args.n_epochs + 1): if (args.model_type == 'DA') and (best_val_acc >= args.DA_embed_train): model.embed.weight.requires_grad = True train_iter.init_epoch() for batch_ind, batch in enumerate(train_iter): model.train() optimizer.zero_grad() out = model(batch) loss = criterion(out, batch.label) loss.backward() clip_grad_norm( filter(lambda p: p.requires_grad, model.parameters()), 10) optimizer.step() if (batch_ind != 0) and (batch_ind % args.dev_every == 0): val_correct, val_loss = evaluate(val_iter, model, criterion) val_accuracy = 100 * val_correct / len(val) print(' Batch Step {}/{}, Val Loss: {:.4f}, Val Accuracy: {:.4f}'.\ format(batch_ind, len(train) // args.batch_size, val_loss, val_accuracy)) train_correct, train_loss = evaluate(train_iter, model, criterion) val_correct, val_loss = evaluate(val_iter, model, criterion) val_accuracy = 100 * val_correct / len(val) val_acc_history.append(val_accuracy) stop_training = early_stop(val_acc_history) if not args.no_comet: experiment.log_metric("Train loss", train_loss) experiment.log_metric("Val loss", val_loss) experiment.log_metric("Accuracy (val)", val_accuracy) experiment.log_metric("Accuracy (train)", 100 * train_correct / len(train)) if args.save_model and (val_accuracy > best_val_acc): best_val_acc = val_accuracy if best_val_acc > 60: snapshot_path = '../saved_models/Model_{}_acc_{:.4f}_epoch_{}_model.pt'.format( args.model_type, val_accuracy, epoch) if args.cuda: torch.save(model.cpu(), snapshot_path) model = model.cuda() else: torch.save(model, snapshot_path) print('Epoch: {}, Train Loss: {:.4f}, Val Loss: {:.4f}, Train Acc: {:.2f}, Val Acc: {:.2f}, Best Val Acc: {:.2f}'.\ format(epoch, train_loss, val_loss, 100 * train_correct / len(train), val_accuracy, best_val_acc)) if stop_training: print('Early stop triggered.') break
'l0': 1365 }, 1: { 'nb_merge': 0, 'l0': 1014 }, 2: { 'nb_merge': 0, 'l0': 717 } } hyper_params = {'learning_rate': 0.0001, 'alphabet_size': 94} hyper_params.update(dict_hyper_params[exp_index]) comet.log_multiple_params(hyper_params) nb_classes = 2 batch_size = 128 num_epochs = 10 save_model_path = '/home/ecd353/NLP_project/experiments/save/models/amazon/exp' + str( exp_index) + '_best.pth.tar' save_pred_path = '/home/ecd353/NLP_project/experiments/save/predictions/amazon/exp' + str( exp_index) + '.txt' ## generate dataset print("generating dataset") data_path = '/home/ecd353/NLP_project/data/amazon/' training_set, validation_set = dataGenerator(data_path + 'train.txt', max_length=hyper_params['l0'])
def test_autoencoder(): (x_train, _), (x_test, _) = mnist.load_data() x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:]))) x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:]))) config = { "encoder": [{ "kwargs": { "activation": "relu", "units": 256 }, "name": "hidden1", "type": "Dense" }, { "name": "batchnorm", "type": "BatchNormalization" }, { "kwargs": { "rate": 0 }, "name": "dropout", "type": "Dropout" }, { "kwargs": { "activation": "sigmoid", }, "name": "latent", "regularizer": { "type": "l1", "value": 0 }, "type": "Dense" }] } latent_dim = 32 latent_shape = (latent_dim, ) input_shape = (x_train.shape[1], ) print(latent_shape) print(input_shape) ae = Autoencoder(config["encoder"], None, input_shape=input_shape, latent_shape=latent_shape, loss="mean_squared_error", optimizer_params=None) #experiment = Experiment(api_key="ac4P1dtMEjJf1d9hIo9CIuSXC", project_name="mnist-autoencode") experiment = Experiment(project_name="MNIST test", api_key="50kNmWUHJrWHz3FlgtpITIsB1") experiment.log_parameter("Experiment name", "Testing ae") experiment.log_multiple_params(config) experiment.log_parameter("Latent dim", latent_shape[0]) ae.fit(x_train, batch_size=1000, epochs=5, validation_data=x_test) predictions = ae.predict(x_test) scores = np.sqrt(((predictions - x_test)**2).mean()) experiment.log_other("scores", scores) print(scores) print(predictions.shape) pred_imgs = predictions.reshape(-1, 28, 28) fig = plt.figure() for i, img in enumerate(pred_imgs[:5]): fig.add_subplot(2, 5, i + 1) plt.imshow(img) plt.axis('off') fig.add_subplot(2, 5, i + 6) plt.imshow(x_test[i].reshape(28, 28)) plt.axis('off') plt.show()
def test_shared_embedding_autoencoder(): (x_train, _), (x_test, _) = mnist.load_data() x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. x_train = x_train.reshape((len(x_train), np.prod(x_train.shape[1:]))) x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:]))) f_idx = np.random.permutation(x_test.shape[1]) n = int(len(f_idx) / 3) print(n) print(x_train[f_idx[n * 0:n * 0 + n]].shape) x_train_list = [x_train[:, f_idx[n * i:n * i + n]] for i in range(3)] x_test_list = [x_test[:, f_idx[n * i:n * i + n]] for i in range(3)] input_shapes = [(d.shape[1], ) for d in x_train_list] config = { "encoder": [{ "kwargs": { "activation": "relu", "units": 256 }, "name": "hidden1", "type": "Dense" }, { "name": "batchnorm", "type": "BatchNormalization" }, { "kwargs": { "rate": 0 }, "name": "dropout", "type": "Dropout" }, { "kwargs": { "activation": "sigmoid", }, "name": "latent", "regularizer": { "type": "l1", "value": 0 }, "type": "Dense" }] } latent_dim = 32 latent_shape = (latent_dim, ) print(latent_shape) print(input_shapes) ae = SharedEmbeddingAutoencoder(config["encoder"], None, input_shapes=input_shapes, latent_shape=latent_shape, loss="mean_squared_error", optimizer_params=None) #experiment = Experiment(api_key="ac4P1dtMEjJf1d9hIo9CIuSXC", project_name="mnist-autoencode") experiment = Experiment(project_name="MNIST test", api_key="50kNmWUHJrWHz3FlgtpITIsB1") experiment.log_parameter("Experiment name", "Testing mae") experiment.log_multiple_params(config) experiment.log_parameter("Latent dim", latent_shape[0]) ae.fit(x_train_list, batch_size=1000, epochs=10, validation_data=x_test_list) predictions = ae.predict(x_test_list) print([p.shape for p in predictions]) #scores = [np.sqrt(((p- x)**2).mean()) for p,x in zip(predictions, x_test_list)] #experiment.log_other("scores", scores) #print(scores) #x_test_list = [x_test[f_idx[n*i:n*i+n]] for i in range(2)] predictions_combined = np.empty_like(x_test) for i in range(3): predictions_combined[:, f_idx[n * i:n * i + n]] = predictions[i] pred_imgs = predictions_combined.reshape(-1, 28, 28) print(pred_imgs.shape) fig = plt.figure() for i, img in enumerate(pred_imgs[:5]): fig.add_subplot(2, 5, i + 1) plt.imshow(img) plt.axis('off') fig.add_subplot(2, 5, i + 6) plt.imshow(x_test[i].reshape(28, 28)) plt.axis('off') plt.show()
def main(): args = get_args() hyperparams = vars(args) if not args.no_comet: experiment = Experiment(api_key="5yzCYxgDmFnt1fhJWTRQIkETT", log_code=True) experiment.log_multiple_params(hyperparams) text_field = data.Field(tokenize=custom_tokenizer, fix_length=args.sentence_len, unk_token='<**UNK**>') label_field = data.Field(sequential=False, unk_token=None) pair_field = data.RawField() print('Loading MultiNLI Dataset') train = get_dataset(text_field, label_field, pair_field, 'train') val = get_dataset(text_field, label_field, pair_field, 'val_matched') text_field.build_vocab(train, max_size=args.max_vocab_size) label_field.build_vocab(train, val) if args.word_vectors: text_field.vocab.load_vectors(args.word_vectors) device = -1 if args.cuda: device = None print('Generating Iterators') train_iter, val_iter = data.BucketIterator.splits( (train, val), batch_size=args.batch_size, shuffle=True, sort_key=sort_key, device=device) train_iter.repeat = False args.n_embed = len(text_field.vocab) args.d_out = len(label_field.vocab) args.n_cells = args.n_layers if args.bidir: args.n_cells *= 2 print(args) if args.load_model: model = torch.load(args.load_model) else: model = MODELS[args.model_type](args) print('Loading Word Embeddings') model.embed.weight.data = text_field.vocab.vectors criterion = nn.CrossEntropyLoss() if args.cuda: model = model.cuda() criterion = criterion.cuda() print(model) val_correct, val_loss = evaluate(val_iter, model, criterion) val_accuracy = 100 * val_correct / len(val) print(val_accuracy)
experiment.log_parameter("Experiment name", "Testing different layers") experiment.log_parameter("Latent dim", latent_shape[0]) groups = data_reader.groups all_scores = [] for i in range(3): ae = Autoencoder(config[i]["encoder"], config[i]["decoder"], input_shape=input_shapes[i], latent_shape=latent_shape, loss="mean_squared_error", optimizer_params=None) experiment.log_multiple_params(config[i]) scores = ae.cross_validate(data[i], groups, experiment=experiment, epochs=10000, n_splits=4, log_prefix=f"dataset_{i}_") all_scores.append(scores) mean_scores = np.mean(scores) experiment.log_metric(f"mean_scores_{i}", mean_scores) experiment.log_other(f"scores_{i}", scores)
class Trainer: def __init__(self): # Simple training script for training a RetinaNet network. # Dataset type, must be one of csv or coco. self.dataset = 'coco' # Path to COCO directory self.coco_path = './data' # Path to file containing training annotations (see readme) self.csv_train = None # Path to file containing class list (see readme) self.csv_classes = None # Path to file containing validation annotations (optional, see readme) self.csv_val = None # Resnet depth, must be one of 18, 34, 50, 101, 152 self.depth = 50 # batch_size self.bs = 8 # learning rate self.lr = 1e-5 # Number of epochs self.epochs = 10 # set device self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') # set focal loss self.focal_loss = losses.FocalLoss() # module calcurating nms self.nms = NMS(BBoxTransform, ClipBoxes) # index of the saving model self.save_name = 2 # use comet_ml self.cml = True # classification_loss self.cls_loss_meter = AverageMeter() # regression_loss self.rgrs_loss_meter = AverageMeter() self.set_comet_ml() def set_comet_ml(self): params = { 'epochs': self.epochs, 'batch_size': self.bs, 'lr': self.lr, 'resnet_depth': self.depth, 'save_name': self.save_name, } if self.cml: self.experiment = Experiment(api_key="xK18bJy5xiPuPf9Dptr43ZuMk", project_name="retinanet-coco", workspace="tanimutomo") else: self.experiment = None if self.cml: self.experiment.log_multiple_params(params) def set_dataset(self): # Create the data loaders if self.dataset == 'coco': if self.coco_path is None: raise ValueError( 'Must provide --coco_path when training on COCO,') dataset_train = CocoDataset(self.coco_path, set_name='train2017', transform=transforms.Compose([ Normalizer(), Augmenter(), Resizer() ])) dataset_val = CocoDataset(self.coco_path, set_name='val2017', transform=transforms.Compose( [Normalizer(), Resizer()])) elif self.dataset == 'csv': if self.csv_train is None: raise ValueError( 'Must provide --csv_train when training on COCO,') if self.csv_classes is None: raise ValueError( 'Must provide --csv_classes when training on COCO,') dataset_train = CSVDataset(train_file=self.csv_train, class_list=self.csv_classes, transform=transforms.Compose([ Normalizer(), Augmenter(), Resizer() ])) if self.csv_val is None: dataset_val = None print('No validation annotations provided.') else: dataset_val = CSVDataset(train_file=self.csv_val, class_list=self.csv_classes, transform=transforms.Compose( [Normalizer(), Resizer()])) else: raise ValueError( 'Dataset type not understood (must be csv or coco), exiting.') return dataset_train, dataset_val def set_models(self, dataset_train): # Create the model if self.depth == 18: retinanet = model.resnet18(num_classes=dataset_train.num_classes(), pretrained=True) elif self.depth == 34: retinanet = model.resnet34(num_classes=dataset_train.num_classes(), pretrained=True) elif self.depth == 50: retinanet = model.resnet50(num_classes=dataset_train.num_classes(), pretrained=True) elif self.depth == 101: retinanet = model.resnet101( num_classes=dataset_train.num_classes(), pretrained=True) elif self.depth == 152: retinanet = model.resnet152( num_classes=dataset_train.num_classes(), pretrained=True) else: raise ValueError( 'Unsupported model depth, must be one of 18, 34, 50, 101, 152') if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs retinanet = nn.DataParallel(retinanet) self.retinanet = retinanet.to(self.device) self.retinanet.training = True self.optimizer = optim.Adam(self.retinanet.parameters(), lr=self.lr) # This lr_shceduler reduce the learning rate based on the models's validation loss self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, patience=3, verbose=True) self.loss_hist = collections.deque(maxlen=500) # self.retinanet.train() # self.retinanet.freeze_bn() def iterate(self): dataset_train, dataset_val = self.set_dataset() sampler = AspectRatioBasedSampler(dataset_train, batch_size=self.bs, drop_last=False) dataloader_train = DataLoader(dataset_train, num_workers=0, collate_fn=collater, batch_sampler=sampler) # if dataset_val is not None: # sampler_val = AspectRatioBasedSampler(dataset_val, batch_size=1, drop_last=False) # dataloader_val = DataLoader(dataset_val, num_workers=0, collate_fn=collater, batch_sampler=sampler_val) print('Num training images: {}'.format(len(dataset_train))) self.set_models(dataset_train) for epoch_num in range(self.epochs): epoch_loss = [] metrics = { 'classification_loss': self.cls_loss_meter.avg, 'regression_loss': self.rgrs_loss_meter.avg, 'entire_loss': self.cls_loss_meter.avg + self.rgrs_loss_meter.avg } if self.experiment is not None: self.experiment.log_multiple_metrics(metrics, step=epoch_num) self.retinanet.train() self.retinanet.module.freeze_bn() epoch_loss = self.train(epoch_num, epoch_loss, dataloader_train) self.retinanet.eval() self.evaluate(epoch_num, dataset_val) torch.save( self.retinanet.state_dict(), os.path.join( './saved_models', 'model{}_final_{}.pth'.format(self.save_name, epoch_num))) # torch.save(self.retinanet.module, '{}_self.retinanet_{}.pt'.format(self.dataset, epoch_num)) # self.retinanet.load_state_dict(torch.load("./saved_models/model_final_0.pth")) self.scheduler.step(np.mean(epoch_loss)) self.retinanet.eval() def train(self, epoch_num, epoch_loss, dataloader_train): for iter_num, data in enumerate(dataloader_train): try: self.optimizer.zero_grad() input = data['img'].to(self.device).float() annot = data['annot'].to(self.device) regression, classification, anchors = self.retinanet(input) classification_loss, regression_loss = self.focal_loss.calcurate( classification, regression, anchors, annot) classification_loss = classification_loss.mean() regression_loss = regression_loss.mean() self.cls_loss_meter.update(classification_loss) self.rgrs_loss_meter.update(regression_loss) loss = classification_loss + regression_loss if bool(loss == 0): continue loss.backward() torch.nn.utils.clip_grad_norm_(self.retinanet.parameters(), 0.1) self.optimizer.step() self.loss_hist.append(float(loss.item())) epoch_loss.append(float(loss.item())) torch.nn.utils.clip_grad_norm_(self.retinanet.parameters(), 0.1) print( 'Epoch: {} | Iteration: {} | Classification loss: {:1.5f} | Regression loss: {:1.5f} | Running loss: {:1.5f}' .format(epoch_num, iter_num, float(classification_loss), float(regression_loss), np.mean(self.loss_hist))) del classification_loss del regression_loss except Exception as e: print(e) continue # if iter_num == 10: # break return epoch_loss def evaluate(self, epoch_num, dataset_val): if self.dataset == 'coco': print('Evaluating dataset') coco_eval.evaluate_coco(dataset_val, self.retinanet, self.nms, self.device) elif self.dataset == 'csv' and self.csv_val is not None: print('Evaluating dataset') mAP = csv_eval.evaluate(dataset_val, self.retinanet)
def main(unused_args): hparams = { 'datasource': FLAGS.datasource, 'num_classes_train': FLAGS.num_classes, 'num_classes_val': FLAGS.num_classes, 'num_classes_test': FLAGS.num_classes_test, 'num_shot_train': FLAGS.num_shot_train, 'num_shot_test': FLAGS.num_shot_test, 'steps': FLAGS.steps, 'meta_batch_size': FLAGS.meta_batch_size, 'meta_lr': FLAGS.meta_lr, 'notes': FLAGS.notes, } hparams = check_default_config(hparams) if FLAGS.train and not FLAGS.load: hparams['mode'] = 'train' save_string = [ hparams['datasource'], str(hparams['num_classes_train']) + 'way', str(hparams['num_shot_train']) + 'shot', strftime('%y%m%d_%H%M'), ] save_folder = '_'.join(map(str, save_string)) + '/' os.makedirs(FLAGS.savepath + save_folder) hparams['savepath'] = FLAGS.savepath + save_folder save_config(hparams, FLAGS.savepath + save_folder) # elif FLAGS.test: # hparams = load_config(FLAGS.savepath + 'config.json', test=True, notes=FLAGS.notes) if FLAGS.comet: experiment = Experiment(api_key=os.environ['COMETML_API_KEY'], project_name='meta') experiment.log_multiple_params(hparams) if FLAGS.train and FLAGS.datasource in [ 'omniglot', 'miniimagenet', 'cifar' ]: num_shot_train = FLAGS.num_shot_train or 1 num_shot_test = FLAGS.num_shot_test or 1 data_generator = DataGenerator( datasource=FLAGS.datasource, num_classes=FLAGS.num_classes, num_samples_per_class=num_shot_train + num_shot_test, batch_size=FLAGS.meta_batch_size, test_set=False, ) # Tensorflow queue for metatraining dataset # metatrain_image_tensor - (batch_size, num_classes * num_samples_per_class, 28 * 28) # metatrain_label_tensor - (batch_size, num_classes * num_samples_per_class, num_classes) metatrain_image_tensor, metatrain_label_tensor = data_generator.make_data_tensor( train=True, load=True, savepath='test.pkl') train_inputs = tf.slice(metatrain_image_tensor, [0, 0, 0], [-1, FLAGS.num_classes * num_shot_train, -1]) test_inputs = tf.slice(metatrain_image_tensor, [0, FLAGS.num_classes * num_shot_train, 0], [-1, -1, -1]) train_labels = tf.slice(metatrain_label_tensor, [0, 0, 0], [-1, FLAGS.num_classes * num_shot_train, -1]) test_labels = tf.slice(metatrain_label_tensor, [0, FLAGS.num_classes * num_shot_train, 0], [-1, -1, -1]) metatrain_input_tensors = { 'train_inputs': train_inputs, # batch_size, num_classes * (num_samples_per_class - update_batch_size), 28 * 28 'train_labels': train_labels, # batch_size, num_classes * (num_samples_per_class - update_batch_size), num_classes 'test_inputs': test_inputs, # batch_size, num_classes * update_batch_size, 28 * 28 'test_labels': test_labels, # batch_size, num_classes * update_batch_size, num_classes } data_generator = DataGenerator( datasource=FLAGS.datasource, num_classes=hparams['num_classes_val'], num_samples_per_class=num_shot_train + num_shot_test, batch_size=16, test_set=False, ) # Tensorflow queue for metavalidation dataset metaval_image_tensor, metaval_label_tensor = data_generator.make_data_tensor( train=False) train_inputs = tf.slice( metaval_image_tensor, [0, 0, 0], [-1, hparams['num_classes_val'] * num_shot_train, -1]) test_inputs = tf.slice( metaval_image_tensor, [0, hparams['num_classes_val'] * num_shot_train, 0], [-1, -1, -1]) train_labels = tf.slice( metaval_label_tensor, [0, 0, 0], [-1, hparams['num_classes_val'] * num_shot_train, -1]) test_labels = tf.slice( metaval_label_tensor, [0, hparams['num_classes_val'] * num_shot_train, 0], [-1, -1, -1]) metaval_input_tensors = { 'train_inputs': train_inputs, # batch_size, num_classes * (num_samples_per_class - update_batch_size), 28 * 28 'train_labels': train_labels, # batch_size, num_classes * (num_samples_per_class - update_batch_size), num_classes 'test_inputs': test_inputs, # batch_size, num_classes * update_batch_size, 28 * 28 'test_labels': test_labels, # batch_size, num_classes * update_batch_size, num_classes } # Graph for metatraining # using scope reuse=tf.AUTO_REUSE, not sure if this is the best way to do it if FLAGS.datasource == 'miniimagenet': # model_metatrain = CNN_MiniImagenet('model', n_way=FLAGS.num_classes, layers=4, input_tensors=metatrain_input_tensors) model_metatrain = CNN_miniimagenet( 'model', num_classes=FLAGS.num_classes, input_tensors=metatrain_input_tensors) elif FLAGS.datasource == 'cifar': model_metatrain = CNN_cifar('model', num_classes=FLAGS.num_classes, input_tensors=metatrain_input_tensors) else: model_metatrain = CNN_omniglot( 'model', num_classes=FLAGS.num_classes, input_tensors=metatrain_input_tensors) # model_metatrain = CNN2('model', n_way=FLAGS.num_classes, layers=4, input_tensors=metatrain_input_tensors) # Graph for metavalidation if FLAGS.datasource == 'miniimagenet': # model_metaval = CNN_MiniImagenet('model', n_way=FLAGS.num_classes, layers=4, input_tensors=metaval_input_tensors) model_metaval = CNN_miniimagenet( 'model', num_classes=hparams['num_classes_val'], input_tensors=metaval_input_tensors) elif FLAGS.datasource == 'cifar': model_metaval = CNN_cifar('model', num_classes=hparams['num_classes_val'], input_tensors=metaval_input_tensors) else: model_metaval = CNN_omniglot('model', num_classes=FLAGS.num_classes, input_tensors=metaval_input_tensors) # model_metaval = CNN2('model', n_way=FLAGS.num_classes, layers=4, input_tensors=metaval_input_tensors) sess = tf.InteractiveSession() tf.global_variables_initializer().run() if FLAGS.load: model_metatrain.load(sess, FLAGS.savepath, verbose=True) model_metaval.load(sess, FLAGS.savepath, verbose=True) tf.train.start_queue_runners() saved_metaval_loss = np.inf steps = FLAGS.steps or 40000 try: for step in np.arange(steps): # metatrain_loss, metatrain_accuracy, _, _ = sess.run([model_metatrain.loss, model_metatrain.test_accuracy, model_metatrain.optimize, model_metatrain.ae_optimize], {model_metatrain.is_training: True}) metatrain_loss, metatrain_accuracy, _ = sess.run( [ model_metatrain.loss, model_metatrain.test_accuracy, model_metatrain.optimize ], {model_metatrain.is_training: True}) if step > 0 and step % FLAGS.print_every == 0: # model_metatrain.writer.add_summary(metatrain_summary, step) print('Step #{} - Loss : {:.3f} - Acc : {:.3f}'.format( step, metatrain_loss, metatrain_accuracy)) if FLAGS.comet: experiment.log_metric("train_loss", metatrain_loss, step=step) experiment.log_metric("train_accuracy", metatrain_accuracy, step=step) if step > 0 and (step % FLAGS.validate_every == 0 or step == (steps - 1)): if step == (steps - 1): print('Training complete!') metaval_loss, metaval_accuracy = sess.run( [model_metaval.loss, model_metaval.test_accuracy], {model_metaval.is_training: False}) # model_metaval.writer.add_summary(metaval_summary, step) print('Validation Results - Loss : {:.3f} - Acc : {:.3f}'. format(metaval_loss, metaval_accuracy)) if FLAGS.comet: experiment.log_metric("val_loss", metaval_loss, step=step) experiment.log_metric("val_accuracy", metaval_accuracy, step=step) if metaval_loss < saved_metaval_loss: saved_metaval_loss = metaval_loss if not FLAGS.load: model_metatrain.save(sess, FLAGS.savepath + save_folder, global_step=step, verbose=True) else: model_metatrain.save(sess, FLAGS.savepath, global_step=step, verbose=True) # Catch Ctrl-C event and allow save option except KeyboardInterrupt: response = raw_input( '\nSave latest model at Step #{}? (y/n)\n'.format(step)) if response == 'y': model_metatrain.save(sess, FLAGS.savepath, global_step=step, verbose=True) else: print('Latest model not saved.') if FLAGS.test and FLAGS.datasource in [ 'omniglot', 'miniimagenet', 'cifar' ]: NUM_TEST_SAMPLES = 600 num_classes_test = FLAGS.num_classes_test or FLAGS.num_classes num_shot_train = FLAGS.num_shot_train or 1 num_shot_test = FLAGS.num_shot_test or 1 data_generator = DataGenerator( datasource=FLAGS.datasource, num_classes=num_classes_test, num_samples_per_class=num_shot_train + num_shot_test, batch_size=1, # use 1 for testing to calculate stdev and ci95 test_set=True, ) image_tensor, label_tensor = data_generator.make_data_tensor( train=False) train_inputs = tf.slice(image_tensor, [0, 0, 0], [-1, num_classes_test * num_shot_train, -1]) test_inputs = tf.slice(image_tensor, [0, num_classes_test * num_shot_train, 0], [-1, -1, -1]) train_labels = tf.slice(label_tensor, [0, 0, 0], [-1, num_classes_test * num_shot_train, -1]) test_labels = tf.slice(label_tensor, [0, num_classes_test * num_shot_train, 0], [-1, -1, -1]) input_tensors = { 'train_inputs': train_inputs, # batch_size, num_classes * (num_samples_per_class - update_batch_size), 28 * 28 'train_labels': train_labels, # batch_size, num_classes * (num_samples_per_class - update_batch_size), num_classes 'test_inputs': test_inputs, # batch_size, num_classes * update_batch_size, 28 * 28 'test_labels': test_labels, # batch_size, num_classes * update_batch_size, num_classes } if FLAGS.datasource == 'miniimagenet': # model = CNN_MiniImagenet('model', n_way=FLAGS.num_classes, layers=4, input_tensors=input_tensors) model = CNN_miniimagenet('model', num_classes=FLAGS.num_classes, input_tensors=input_tensors) elif FLAGS.datasource == 'cifar': model = CNN_cifar('model', num_classes=FLAGS.num_classes, input_tensors=input_tensors) else: model = CNN_omniglot('model', num_classes=FLAGS.num_classes, input_tensors=input_tensors) # model = CNN2('model', n_way=FLAGS.num_classes, layers=4, input_tensors=input_tensors) sess = tf.InteractiveSession() tf.global_variables_initializer().run() model.load(sess, FLAGS.savepath, verbose=True) tf.train.start_queue_runners() # BEGIN PLOT if FLAGS.plot: activations, labels = sess.run( [model.train_features, model.train_labels], {model.is_training: False}) activations = activations.reshape( [num_shot_train * FLAGS.num_classes, -1]) from sklearn.manifold import TSNE from sklearn.decomposition import PCA pca = PCA(50) print('Compressing with PCA...') activations_50dim = pca.fit_transform(activations) tsne = TSNE() print('Compressing with tSNE...') activations_2dim = tsne.fit_transform(activations_50dim) labels = np.argmax(labels, axis=1) fig, ax = plt.subplots() for i in np.arange(FLAGS.num_classes): ax.scatter(activations_2dim[np.where(labels == i)][:, 0], activations_2dim[np.where(labels == i)][:, 1], s=5.) plt.show() quit() # END PLOT accuracy_list = [] for task in np.arange(NUM_TEST_SAMPLES): accuracy = sess.run(model.test_accuracy, {model.is_training: False}) accuracy_list.append(accuracy) if task > 0 and task % 100 == 0: print('Metatested on {} tasks...'.format(task)) avg = np.mean(accuracy_list) stdev = np.std(accuracy_list) ci95 = 1.96 * stdev / np.sqrt(NUM_TEST_SAMPLES) print('\nEnd of Test!') print('Accuracy : {:.4f}'.format(avg)) print('StdDev : {:.4f}'.format(stdev)) print('95% Confidence Interval : {:.4f}'.format(ci95)) if FLAGS.comet: experiment.log_metric("test_accuracy_mean", avg) experiment.log_metric("test_accuracy_stdev", stdev) experiment.log_metric("test_accuracy_ci95", ci95) if FLAGS.train and FLAGS.datasource in ['sinusoid', 'multimodal', 'step']: num_shot_train = FLAGS.num_shot_train or 10 num_shot_test = FLAGS.num_shot_test or 10 data_generator = DataGenerator( datasource=FLAGS.datasource, num_classes=None, num_samples_per_class=num_shot_train + num_shot_test, batch_size=FLAGS.meta_batch_size, test_set=None, ) model = FFN('model') sess = tf.InteractiveSession() tf.global_variables_initializer().run() saved_loss = np.inf steps = FLAGS.steps or 50000 try: for step in np.arange(steps): if FLAGS.datasource == 'multimodal': batch_x, batch_y, amp, phase, slope, intercept, modes = data_generator.generate( ) amp = amp * modes + (modes == False).astype(np.float32) elif FLAGS.datasource == 'step': batch_x, batch_y, start_step = data_generator.generate() amp = np.ones(batch_x.shape[0]) else: batch_x, batch_y, amp, phase = data_generator.generate() amp = np.ones(batch_x.shape[0]) train_inputs = batch_x[:, :num_shot_train, :] train_labels = batch_y[:, :num_shot_train, :] test_inputs = batch_x[:, num_shot_train:, :] test_labels = batch_y[:, num_shot_train:, :] feed_dict = { model.train_inputs: train_inputs, model.train_labels: train_labels, model.test_inputs: test_inputs, model.test_labels: test_labels, model.amp: amp, # use amplitude to scale loss } metatrain_postloss, _ = sess.run([model.loss, model.optimize], feed_dict) if step > 0 and step % FLAGS.print_every == 0: # model.writer.add_summary(metatrain_summary, step) print('Step #{} - PreLoss : {:.3f} - PostLoss : {:.3f}'. format(step, 0., metatrain_postloss)) if step == (steps - 1): print('Training complete!') if metatrain_postloss < saved_loss: saved_loss = metatrain_postloss model.save(sess, FLAGS.savepath + save_folder, global_step=step, verbose=True) # Catch Ctrl-C event and allow save option except KeyboardInterrupt: response = raw_input( '\nSave latest model at Step #{}? (y/n)\n'.format(step)) if response == 'y': model.save(sess, FLAGS.savepath, global_step=step, verbose=True) else: print('Latest model not saved.') if FLAGS.test and FLAGS.datasource in ['sinusoid', 'multimodal', 'step']: num_shot_train = FLAGS.num_shot_train or 10 data_generator = DataGenerator( datasource=FLAGS.datasource, num_classes=None, num_samples_per_class=num_shot_train, batch_size=1, test_set=None, ) model = FFN('model', num_train_samples=num_shot_train, num_test_samples=50) sess = tf.InteractiveSession() model.load(sess, FLAGS.savepath, verbose=True) if FLAGS.datasource == 'multimodal': train_inputs, train_labels, amp, phase, slope, intercept, modes = data_generator.generate( ) amp = amp * modes + (modes == False).astype(np.float32) x = np.arange(-5., 5., 0.2) if modes[0] == 0: y = slope * x + intercept else: y = amp * np.sin(x - phase) elif FLAGS.datasource == 'step': train_inputs, train_labels, start_step = data_generator.generate() x = np.arange(-5., 5., 0.2) y = np.ones_like(x) - (x < start_step).astype( np.float32) - (x > (start_step + 2)).astype(np.float32) else: train_inputs, train_labels, amp, phase = data_generator.generate() amp = 5. phase = 0. x = np.arange(5., 15., 0.2).reshape(1, -1, 1) y = amp * np.sin(x - phase).reshape(1, -1, 1) train_inputs = np.arange(5., 10., .5).reshape(1, -1, 1) # train_inputs = np.arange(-5., 0., .5).reshape(1, -1, 1) train_labels = amp * np.sin(train_inputs - phase) feed_dict = { model.train_inputs: train_inputs, model.train_labels: train_labels, model.test_inputs: x.reshape(1, -1, 1), model.test_labels: y.reshape(1, -1, 1), } postprediction, postloss = sess.run( [model.predictions, model.plain_loss], feed_dict) print(postloss) fig, ax = plt.subplots() ax.plot(x.reshape(-1), y.reshape(-1), color='#2c3e50', linewidth=0.8, label='Truth') ax.scatter(train_inputs.reshape(-1), train_labels.reshape(-1), color='#2c3e50', label='Training Set') ax.plot(x.reshape(-1), postprediction.reshape(-1), label='Prediction', color='#e74c3c', linestyle='--') ax.legend() ax.set_title(postloss) plt.show() if FLAGS.filename is not None: fig.savefig('figures/' + FLAGS.filename + '.png', dpi=72, bbox_inches='tight')