def _generate_eval_video(self): env_main, env1, env2 = self._env_func() with tempfile.TemporaryDirectory(dir="/tmp") as temp: path_to_video = os.path.join(temp, "movie.mp4") PygameInteractiveEnvRecorder(env=env_main, fps=60, win_size=(640 * 1.5, 480 * 1.5), p1={ "policy": self.model, "env": env1, "frameskip": self._params["frameskip"], }, p2={ "policy": "human", "env": env2, "frameskip": 1, }, record_output_path=path_to_video, render_n_frames_after_done=250, render=False).run() encoded = base64.b64encode(open(path_to_video, "rb").read()) html = f'<video controls><source type="video/mp4" ' \ f'src="data:video/mp4;base64,{encoded.decode("utf-8")}"></video>' open(path_to_video, "w+").write(html) neptune.send_artifact(path_to_video, f"movies/movie_{self._iteration}_{self._context.num_timesteps}.html")
def main(): neptune.init(api_token=os.getenv('NEPTUNE_API_TOKEN'), project_qualified_name=os.getenv('NEPTUNE_PROJECT')) application_table_path = os.path.join(RAW_DATA_DIRPATH, 'application_train.csv.zip') application_table = pd.read_csv(application_table_path, nrows=NROWS) index_table = application_table[['SK_ID_CURR', 'TARGET']] with neptune.create_experiment(name='validation schema', tags=['processed', 'validation'], upload_source_files=get_filepaths()): train_idx, valid_idx = train_test_split(index_table, test_size=TEST_SIZE, random_state=SEED) train_idx_path = os.path.join(INTERIM_FEATURES_DIRPATH, 'train_idx.csv') train_idx.to_csv(train_idx_path, index=None) neptune.send_artifact(train_idx_path) neptune.set_property('train_split_version', md5_hash(train_idx_path)) valid_idx_path = os.path.join(INTERIM_FEATURES_DIRPATH, 'valid_idx.csv') valid_idx.to_csv(valid_idx_path, index=None) neptune.send_artifact(valid_idx_path) neptune.set_property('valid_split_version', md5_hash(valid_idx_path))
def handle_directories(self): exp = neptune.get_experiment() # download_artifacts neptune.send_artifact(self.data_dir) if self._api_version == 1: with self.with_check_if_file_appears("output.zip"): exp.download_artifacts() else: with self.with_assert_raises( DownloadArtifactsUnsupportedException): exp.download_artifacts() # create some nested artifacts neptune.log_artifact(self.img_path, destination="main dir/sub dir/art1") neptune.log_artifact(self.img_path, destination="main dir/sub dir/art2") neptune.log_artifact(self.img_path, destination="main dir/sub dir/art3") # downloading artifact - download_artifact # non existing artifact if self._api_version == 1: with self.with_assert_raises(FileNotFound): exp.download_artifact("main dir/sub dir/art100") else: with self.with_assert_raises(DownloadArtifactUnsupportedException): exp.download_artifact("main dir/sub dir/art100") # artifact directories if self._api_version == 1: with self.with_assert_raises(HTTPError): exp.download_artifact("main dir/sub dir") else: with self.with_assert_raises(DownloadArtifactUnsupportedException): exp.download_artifact("main dir/sub dir") # deleting artifacts neptune.delete_artifacts("main dir/sub dir/art1") # delete non existing artifact if self._api_version == 1: neptune.delete_artifacts("main dir/sub dir/art100") else: with self.with_assert_raises( DeleteArtifactUnsupportedInAlphaException): neptune.delete_artifacts("main dir/sub dir/art100") # delete dir if self._api_version == 1: neptune.delete_artifacts("main dir/sub dir") else: with self.with_assert_raises( DeleteArtifactUnsupportedInAlphaException): neptune.delete_artifacts("main dir/sub dir")
def print_table_input_to_message(self): table_data = [['x'] + list(range(self.opts.n_features)) ] + [[i] + [None] * self.opts.n_features for i in range(self.opts.n_features)] for (input1, input2), messages in self.input_to_message.items(): table_data[input1 + 1][input2 + 1] = ' '.join((' '.join( (str(s) for s in message)) for message in set(messages))) for a, b in self.test_targets: table_data[a + 1][b + 1] = '*' + table_data[a + 1][b + 1] filename = f'{self.prefix}input_to_message_{self.epoch_counter}.txt' with open(file=filename, mode='w', encoding='utf-8') as file: file.write(tabulate(table_data, tablefmt='fancy_grid')) neptune.send_artifact(filename)
def build_experiment(conf, logpath, resultname, csvfile): with open(csvfile) as stats_f: params = stats_f.readline().strip().split(",") stats = stats_f.readlines() for s in stats: elems = s.strip().split(",") counter, scalar_fit, priority_fit = [elems[0], elems[4], elems[5]] print(counter, scalar_fit, priority_fit) exp_config = read_toml(conf) exp_number = 1 champ = read_berb_log(logpath) exp_name = champ['chromosome']['name'] exp_desc = str(champ['tag']) exp_params = { "some_param": 0.1, "other_param": 128, "yet_another_param": 31337 } exp_log_artifact = ["data/champion_statistics.csv", "mean_statistics.csv"] #Neptune init neptune.init('special-circumstances/sandbox', api_token=None) neptune.create_experiment(name=exp_name, params=exp_params) for s in stats: elems = s.strip().split(",") counter, scalar_fit, priority_fit = [elems[0], elems[4], elems[5]] neptune.log_metric(params[0], int(counter)) neptune.log_metric(params[4], float(scalar_fit)) neptune.log_metric(params[5], float(priority_fit)) neptune.log_image( 'pleasures_1', "/home/armadilo/projects/neptune/data/clamp-liked-zeros-count-pleasures.png" ) neptune.log_image( 'pleasures_2', "/home/armadilo/projects/neptune/data/lamas-koala-zero-count-pleasures.png" ) neptune.send_artifact( '/home/armadilo/projects/neptune/data/champion_statistics.csv') neptune.send_artifact( '/home/armadilo/projects/neptune/data/mean_statistics.csv')
def visualize_embeddings(self): embeddings = self.game.receiver.embedding.weight.detach().transpose( 1, 0) pca = PCA(n_components=2) embeddings_projected = pca.fit_transform(embeddings) np.savetxt('embs.txt', embeddings_projected) neptune.send_artifact('embs.txt') ax = sns.scatterplot(x=embeddings_projected[:, 0], y=embeddings_projected[:, 1]) for i in range(10): ax.annotate(str(i), embeddings_projected[i], size=20) sns.despine(left=True, bottom=True) plt.xlabel('First principal component') plt.ylabel('Second principal component') figure = ax.get_figure() send_figure(figure, channel_name='embeddings') figure.savefig('figx.png') plt.close(figure)
def handle_files_and_images(self): # image # `image_name` and `description` will be lost (`send_image` the same as `log_image`) neptune.send_image("image", self.img_path, name="name", description="desc") # artifact with default dest neptune.send_artifact(self.text_file_path) exp = neptune.get_experiment() with self.with_check_if_file_appears("text.txt"): exp.download_artifact("text.txt") with self.with_check_if_file_appears("custom_dest/text.txt"): exp.download_artifact("text.txt", "custom_dest") # artifact with custom dest neptune.send_artifact(self.text_file_path, destination="something.txt") exp = neptune.get_experiment() with self.with_check_if_file_appears("something.txt"): exp.download_artifact("something.txt") with self.with_check_if_file_appears("custom_dest/something.txt"): exp.download_artifact("something.txt", "custom_dest") # destination dirs neptune.log_artifact(self.text_file_path, destination="dir/text file artifact") neptune.log_artifact(self.text_file_path, destination="dir/artifact_to_delete") # deleting neptune.delete_artifacts("dir/artifact_to_delete") # streams with open(self.text_file_path, mode="r") as f: neptune.send_artifact(f, destination="file stream.txt")
'search_hyperopt_tpe.py', 'search_hyperopt_basic.py', 'utils.py' ]): trials = Trials() _ = fmin(objective, SPACE, trials=trials, algo=tpe.suggest, **HPO_PARAMS) results = hpo_utils.hyperopt2skopt(trials, SPACE) best_auc = -1.0 * results.fun best_params = results.x # log metrics print('Best Validation AUC: {}'.format(best_auc)) print('Best Params: {}'.format(best_params)) neptune.send_metric('validation auc', best_auc) # log results joblib.dump(trials, 'artifacts/hyperopt_trials.pkl') joblib.dump(results, 'artifacts/hyperopt_results.pkl') joblib.dump(SPACE, 'artifacts/hyperopt_space.pkl') neptune.send_artifact('artifacts/hyperopt_trials.pkl') neptune.send_artifact('artifacts/hyperopt_results.pkl') neptune.send_artifact('artifacts/hyperopt_space.pkl') # log runs sk_utils.send_runs(results) sk_utils.send_best_parameters(results) sk_utils.send_plot_convergence(results, channel_name='diagnostics') sk_utils.send_plot_evaluations(results, channel_name='diagnostics')
def main(): config_path = args.conf initial_weights = args.weights with open(config_path) as config_buffer: config = json.loads(config_buffer.read()) train_set, valid_set, classes = data.create_training_instances(config['train']['train_folder'], None, config['train']['cache_name'], config['model']['labels']) num_classes = len(classes) print('Readed {} classes: {}'.format(num_classes, classes)) train_generator = gen.BatchGenerator( instances=train_set, labels=classes, batch_size=config['train']['batch_size'], input_sz=config['model']['infer_shape'], shuffle=True, norm=data.normalize ) valid_generator = gen.BatchGenerator( instances=valid_set, labels=classes, batch_size=config['train']['batch_size'], input_sz=config['model']['infer_shape'], norm=data.normalize, infer=True ) early_stop = EarlyStopping( monitor='val_loss', min_delta=0, patience=20, mode='min', verbose=1 ) reduce_on_plateau = ReduceLROnPlateau( monitor='val_loss', factor=0.5, patience=5, verbose=1, mode='min', min_delta=0.01, cooldown=0, min_lr=0 ) net_input_shape = (config['model']['infer_shape'][0], config['model']['infer_shape'][1], 3) train_model = models.create( base_name=config['model']['base'], num_classes=num_classes, input_shape=net_input_shape) if initial_weights: train_model.load_weights(initial_weights) print(train_model.summary()) # plot_model(train_model, to_file='images/MobileNetv2.png', show_shapes=True) optimizer = Adam(lr=config['train']['learning_rate'], clipnorm=0.001) train_model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) checkpoint_name = utils.get_checkpoint_name(config) utils.makedirs_4_file(checkpoint_name) static_chk_name = utils.get_static_checkpoint_name(config) utils.makedirs_4_file(static_chk_name) checkpoint_vloss = cbs.CustomModelCheckpoint( model_to_save=train_model, filepath=checkpoint_name, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1 ) neptune_mon = cbs.NeptuneMonitor( monitoring=['loss', 'val_loss', 'accuracy', 'val_accuracy'], neptune=neptune ) chk_static = ModelCheckpoint( filepath=static_chk_name, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1 ) callbacks = [early_stop, reduce_on_plateau, checkpoint_vloss, neptune_mon, chk_static] ### NEPTUNE ### sources_to_upload = [ 'models.py', 'config.json' ] params = { 'infer_size': "H{}xW{}".format(*config['model']['infer_shape']), 'classes': config['model']['labels'], } neptune.create_experiment( name=utils.get_neptune_name(config), upload_stdout=False, upload_source_files=sources_to_upload, params=params ) ### NEPTUNE ### hist = train_model.fit_generator( generator=train_generator, steps_per_epoch=len(train_generator) * config['train']['train_times'], validation_data=valid_generator, validation_steps=len(valid_generator) * config['valid']['valid_times'], epochs=config['train']['nb_epochs'], verbose=2 if config['train']['debug'] else 1, callbacks=callbacks, workers=multiprocessing.cpu_count(), max_queue_size=100 ) neptune.send_artifact(static_chk_name) neptune.send_artifact('config.json')
**HPO_PARAMS) best_auc = -1.0 * results.fun best_params = results.x # log metrics print('Best Validation AUC: {}'.format(best_auc)) print('Best Params: {}'.format(best_params)) neptune.send_metric('validation auc', best_auc) neptune.set_property('best_params', str(to_named_params(best_params))) # log results skopt.dump(results, 'artifacts/gp_results.pkl') joblib.dump(SPACE, 'artifacts/gp_space.pkl') neptune.send_artifact('artifacts/gp_results.pkl') neptune.send_artifact('artifacts/gp_space.pkl') # log diagnostic plots fig, ax = plt.subplots(figsize=(16, 12)) skopt.plots.plot_convergence(results, ax=ax) fig.savefig('plots/gp_convergence.png') neptune.send_image('diagnostics', 'plots/gp_convergence.png') axes = skopt.plots.plot_evaluations(results) fig = axes2fig(axes, figsize=(16, 12)) fig.savefig('plots/gp_evaluations.png') neptune.send_image('diagnostics', 'plots/gp_evaluations.png')
**HPO_PARAMS) best_auc = -1.0 * results.fun best_params = results.x # log metrics print('Best Validation AUC: {}'.format(best_auc)) print('Best Params: {}'.format(best_params)) neptune.send_metric('validation auc', best_auc) neptune.set_property('best_params', str(to_named_params(best_params))) # log results skopt.dump(results, 'artifacts/forest_results.pkl') joblib.dump(SPACE, 'artifacts/forest_space.pkl') neptune.send_artifact('artifacts/forest_results.pkl') neptune.send_artifact('artifacts/forest_space.pkl') # log diagnostic plots fig, ax = plt.subplots(figsize=(16, 12)) skopt.plots.plot_convergence(results, ax=ax) fig.savefig('plots/forest_convergence.png') neptune.send_image('diagnostics', 'plots/forest_convergence.png') axes = skopt.plots.plot_evaluations(results) fig = axes2fig(axes, figsize=(16, 12)) fig.savefig('plots/forest_evaluations.png') neptune.send_image('diagnostics', 'plots/forest_evaluations.png')
def start_train( config, config_path, yolo_model: yolo.YOLO_Model, train_generator, valid_generator, dry_mode: bool ): print('Full training') ############################### # Optimizers ############################### optimizers = { 'sgd': opt.SGD(lr=config['train']['learning_rate']), 'adam': opt.Adam(lr=config['train']['learning_rate']), 'adamax': opt.Adamax(lr=config['train']['learning_rate']), 'nadam': opt.Nadam(lr=config['train']['learning_rate']), 'rmsprop': opt.RMSprop(lr=config['train']['learning_rate']), # 'Radam': RAdam(lr=config['train']['learning_rate'], warmup_proportion=0.1, min_lr=1e-5) } optimizer = optimizers[config['train']['optimizer'].lower()] if config['train']['clipnorm'] > 0: optimizer.clipnorm = config['train']['clipnorm'] if config['train'].get('lr_decay', 0) > 0: optimizer.decay = config['train']['lr_decay'] if config['train']['optimizer'] == 'Nadam': # Just to set field optimizer.decay = 0.0 ############################### # Callbacks ############################### checkpoint_name = utils.get_checkpoint_name(config) utils.makedirs_4_file(checkpoint_name) checkpoint_vloss = cbs.CustomModelCheckpoint( model_to_save=yolo_model.infer_model, filepath=checkpoint_name, monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1 ) # tensorboard_logdir = utils.get_tensorboard_name(config) # utils.makedirs(tensorboard_logdir) # print('Tensorboard dir: {}'.format(tensorboard_logdir)) # tensorboard_cb = TensorBoard( # log_dir=tensorboard_logdir, # histogram_freq=0, # write_graph=False # ) mAP_checkpoint_name = utils.get_mAP_checkpoint_name(config) mAP_checkpoint_static_name = utils.get_mAP_checkpoint_static_name(config) utils.makedirs_4_file(mAP_checkpoint_name) map_evaluator_cb = cbs.MAP_evaluation( model=yolo_model, generator=valid_generator, save_best=True, save_name=mAP_checkpoint_name, save_static_name=mAP_checkpoint_static_name, # tensorboard=tensorboard_cb, neptune=neptune if not dry_mode else None ) reduce_on_plateau = ReduceLROnPlateau( monitor='val_loss', factor=0.4, patience=20, verbose=1, mode='min', min_delta=0, cooldown=10, min_lr=1e-8 ) early_stop = EarlyStopping( monitor='val_loss', min_delta=0, patience=80, mode='min', verbose=1 ) neptune_mon = cbs.NeptuneMonitor( monitoring=['loss', 'val_loss'], neptune=neptune ) # logger_cb = cbs.CustomLogger( # config=config, # tensorboard=tensorboard_cb # ) # fps_logger = cbs.FPSLogger( # infer_model=yolo_model.infer_model, # generator=valid_generator, # infer_sz=config['model']['infer_shape'], # tensorboard=tensorboard_cb # ) callbacks = [ # tensorboard_cb, map_evaluator_cb, # early_stop, reduce_on_plateau, ] ############################### # Prepare fit ############################### if not dry_mode: callbacks.append(neptune_mon) with open('config.json', 'w') as f: json.dump(config, f, indent=4) sources_to_upload = [ 'yolo.py', '_common/backend.py', 'config.json' ] params = { 'base_params': str(config['model']['base_params']), 'infer_size': "H{}xW{}".format(*config['model']['infer_shape']), 'anchors_per_output': config['model']['anchors_per_output'], 'anchors': str(config['model']['anchors']) } tags = [ config['model']['base'] ] logger.info('Tags: {}'.format(tags)) neptune.create_experiment( name=utils.get_neptune_name(config), upload_stdout=False, upload_source_files=sources_to_upload, params=params, tags=tags ) else: config['train']['nb_epochs'] = 10 yolo_model.train_model.compile(loss=yolo.dummy_loss, optimizer=optimizer) yolo_model.train_model.fit_generator( generator=train_generator, steps_per_epoch=len(train_generator) * config['train']['train_times'], validation_data=valid_generator, validation_steps=len(valid_generator) * config['valid']['valid_times'], epochs=config['train']['nb_epochs'], verbose=1, callbacks=callbacks, workers=mp.cpu_count(), max_queue_size=100, use_multiprocessing=False ) if not dry_mode: neptune.send_artifact(mAP_checkpoint_static_name) neptune.send_artifact('config.json')
}): results = skopt.forest_minimize(objective, SPACE, callback=[monitor], **HPO_PARAMS) best_auc = -1.0 * results.fun best_params = results.x neptune.send_metric('valid_auc', best_auc) neptune.set_property('best_params', str(to_named_params(best_params))) # log results skopt.dump(results, os.path.join(REPORTS_DIRPATH, 'skopt_results.pkl')) neptune.send_artifact( os.path.join(REPORTS_DIRPATH, 'skopt_results.pkl')) # log diagnostic plots fig, ax = plt.subplots(figsize=(16, 12)) skopt.plots.plot_convergence(results, ax=ax) fig.savefig(os.path.join(REPORTS_DIRPATH, 'convergence.png')) neptune.send_image('diagnostics', os.path.join(REPORTS_DIRPATH, 'convergence.png')) axes = skopt.plots.plot_evaluations(results) fig = plt.figure(figsize=(16, 12)) fig = axes2fig(axes, fig) fig.savefig(os.path.join(REPORTS_DIRPATH, 'evaluations.png')) neptune.send_image('diagnostics', os.path.join(REPORTS_DIRPATH, 'evaluations.png'))
**HPO_PARAMS) best_auc = -1.0 * results.fun best_params = results.x # log metrics print('Best Validation AUC: {}'.format(best_auc)) print('Best Params: {}'.format(best_params)) neptune.send_metric('validation auc', best_auc) neptune.set_property('best_params', str(to_named_params(best_params))) # log results skopt.dump(results, 'artifacts/random_results.pkl') joblib.dump(SPACE, 'artifacts/random_space.pkl') neptune.send_artifact('artifacts/random_results.pkl') neptune.send_artifact('artifacts/random_space.pkl') # log diagnostic plots fig, ax = plt.subplots(figsize=(16, 12)) skopt.plots.plot_convergence(results, ax=ax) fig.savefig('plots/random_convergence.png') neptune.send_image('diagnostics', 'plots/random_convergence.png') axes = skopt.plots.plot_evaluations(results) fig = axes2fig(axes, figsize=(16, 12)) fig.savefig('plots/random_evaluations.png') neptune.send_image('diagnostics', 'plots/random_evaluations.png')
def main(): print('loading data') train_features_path = os.path.join( FEATURES_DATA_PATH, 'train_features_' + FEATURE_NAME + '.csv') test_features_path = os.path.join(FEATURES_DATA_PATH, 'test_features_' + FEATURE_NAME + '.csv') print('... train') train = pd.read_csv(train_features_path, nrows=TRAINING_PARAMS['nrows']) X = train.sort_values('TransactionDT').drop( ['isFraud', 'TransactionDT', 'TransactionID'], axis=1) y = train.sort_values('TransactionDT')['isFraud'] train = train[["TransactionDT", 'TransactionID']] print('... test') test = pd.read_csv(test_features_path, nrows=TRAINING_PARAMS['nrows']) X_test = test.sort_values('TransactionDT').drop( ['TransactionDT', 'TransactionID'], axis=1) test = test[["TransactionDT", 'TransactionID']] folds = KFold(n_splits=VALIDATION_PARAMS['n_splits'], random_state=VALIDATION_PARAMS['validation_seed']) hyperparams = {**MODEL_PARAMS, **TRAINING_PARAMS, **VALIDATION_PARAMS} print('starting experiment') with neptune.create_experiment( name='model training', params=hyperparams, upload_source_files=get_filepaths(), tags=[MODEL_NAME, 'features_'.format(FEATURE_NAME), 'training']): print('logging data version') log_data_version(train_features_path, prefix='train_features_') log_data_version(test_features_path, prefix='test_features_') print('training') in_fold, out_of_fold, test_preds = fit_predict(X, y, X_test, folds, MODEL_PARAMS, TRAINING_PARAMS) print('logging metrics') train_auc, valid_auc = roc_auc_score(y, in_fold), roc_auc_score( y, out_of_fold) neptune.send_metric('train_auc', train_auc) neptune.send_metric('valid_auc', valid_auc) send_binary_classification_report( y, fmt_preds(out_of_fold), channel_name='valid_classification_report') print('postprocessing predictions') train_predictions_path = os.path.join( PREDICTION_DATA_PATH, 'train_prediction_{}_{}.csv'.format(FEATURE_NAME, MODEL_NAME)) test_predictions_path = os.path.join( PREDICTION_DATA_PATH, 'test_prediction_{}_{}.csv'.format(FEATURE_NAME, MODEL_NAME)) submission_path = os.path.join( PREDICTION_DATA_PATH, 'submission_{}_{}.csv'.format(FEATURE_NAME, MODEL_NAME)) submission = pd.read_csv(SAMPLE_SUBMISSION_PATH) train = pd.concat( [train, pd.DataFrame(out_of_fold, columns=['prediction'])], axis=1) test = pd.concat( [test, pd.DataFrame(test_preds, columns=['prediction'])], axis=1) submission['isFraud'] = pd.merge(submission, test, on='TransactionID')['prediction'] train.to_csv(train_predictions_path, index=None) test.to_csv(test_predictions_path, index=None) submission.to_csv(submission_path, index=None) neptune.send_artifact(train_predictions_path) neptune.send_artifact(test_predictions_path) neptune.send_artifact(submission_path) print('experiment finished')
dev_tensor_attention_masks, dev_tensor_token_type_ids, dev_list_span_idx) metrics = validation.do_validation() model.train() record_eval_metric(neptune, metrics) curr_f1 = metrics['joint_f1'] if curr_f1 > best_eval_f1: best_eval_f1 = curr_f1 model.save_pretrained(model_path) # Calculate the average loss over all of the batches. avg_train_loss = total_train_loss / len(train_dataloader) # Measure how long this epoch took. training_time = format_time(time.time() - t0) print("") print(" Average training loss: {0:.2f}".format(avg_train_loss)) print(" Training epoch took: {:}".format(training_time)) print("") print("Training complete!") print("Total training took {:} (h:mm:ss)".format( format_time(time.time() - total_t0))) # create a zip file for the folder of the model zipdir(model_path, os.path.join(model_path, 'checkpoint.zip')) # upload the model to neptune neptune.send_artifact(os.path.join(model_path, 'checkpoint.zip'))
def save_exp_params(params): with tempfile.TemporaryDirectory(dir="/tmp") as temp: path_to_pickle = os.path.join(temp, params_pickle_name) pickle.dump(params, open(path_to_pickle, "wb")) neptune.send_artifact(path_to_pickle, params_pickle_name)
def main(): neptune.init(api_token=os.getenv('NEPTUNE_API_TOKEN'), project_qualified_name=os.getenv('NEPTUNE_PROJECT')) train_idx = pd.read_csv(TRAIN_IDX_PATH, nrows=NROWS) valid_idx = pd.read_csv(VALID_IDX_PATH, nrows=NROWS) features = pd.read_csv(FEATURES_PATH, nrows=NROWS) train = pd.merge(train_idx, features, on='SK_ID_CURR') valid = pd.merge(valid_idx, features, on='SK_ID_CURR') all_params = { 'num_boost_round': NUM_BOOST_ROUND, 'early_stopping_rounds': EARLY_STOPPING_ROUNDS, **LGBM_PARAMS } with neptune.create_experiment(name='model training', params=all_params, tags=['lgbm'], upload_source_files=get_filepaths(), properties={ 'features_path': FEATURES_PATH, 'features_version': md5_hash(FEATURES_PATH), 'train_split_version': md5_hash(TRAIN_IDX_PATH), 'valid_split_version': md5_hash(VALID_IDX_PATH), }): results = train_evaluate(train, valid, LGBM_PARAMS, callbacks=[neptune_monitor()]) train_score, valid_score = results['train_score'], results[ 'valid_score'] train_preds, valid_preds = results['train_preds'], results[ 'valid_preds'] neptune.send_metric('train_auc', train_score) neptune.send_metric('valid_auc', valid_score) train_pred_path = os.path.join(PREDICTION_DIRPATH, 'train_preds.csv') train_preds.to_csv(train_pred_path, index=None) neptune.send_artifact(train_pred_path) valid_pred_path = os.path.join(PREDICTION_DIRPATH, 'valid_preds.csv') valid_preds.to_csv(valid_pred_path, index=None) neptune.send_artifact(valid_pred_path) model_path = os.path.join(MODEL_DIRPATH, 'model.pkl') joblib.dump(results['model'], model_path) neptune.set_property('model_path', model_path) neptune.set_property('model_version', md5_hash(model_path)) neptune.send_artifact(model_path) if PACKAGE_TO_PROD: saved_path = CreditDefaultClassifier.pack( model=results['model']).save(PRODUCTION_DIRPATH) neptune.set_property('production_model_path', saved_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_confusion_matrix(valid_preds['TARGET'], valid_preds['preds_pos'] > 0.5, ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'conf_matrix.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_roc(valid_preds['TARGET'], valid_preds[['preds_neg', 'preds_pos']], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'roc_auc.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_precision_recall( valid_preds['TARGET'], valid_preds[['preds_neg', 'preds_pos']], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'prec_recall.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) plot_prediction_distribution(valid_preds['TARGET'], valid_preds['preds_pos'], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'preds_dist.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path)