def modify_tags(self): neptune.append_tags("tag1") neptune.append_tag(["tag2_to_remove", "tag3"]) neptune.remove_tag("tag2_to_remove") neptune.remove_tag("tag4_remove_non_existing") exp = neptune.get_experiment() assert set(exp.get_tags()) == { "initial tag 1", "initial tag 2", "tag1", "tag3" }
def do_main(): neptune.init('ods/wheat') # Create experiment with defined parameters neptune.create_experiment(name=model_name, params=PARAMS, tags=[experiment_name, experiment_tag], upload_source_files=[os.path.basename(__file__)]) neptune.append_tags(f'fold_{fold}') device = torch.device(f'cuda:{gpu_number}') if torch.cuda.is_available( ) else torch.device('cpu') print(device) print(len(train_boxes_df)) print(len(train_images_df)) # Leave only > 0 print('Leave only train images with boxes (validation)') with_boxes_filter = train_images_df[image_id_column].isin( train_boxes_df[image_id_column].unique()) negative_images = enumerate_images(DIR_NEGATIVE) negative_images = [(negative_prefix + filename[:-4]) for filename in negative_images] negative_images.sort() # take first 100 now... negative_images = negative_images[:100] """ spike_images = enumerate_images(DIR_SPIKE) spike_images = [(spike_dataset_prefix + filename[:-4]) for filename in spike_images] spike_images.sort() assert len(spike_images) > 0 """ config = get_efficientdet_config('tf_efficientdet_d5') net = EfficientDet(config, pretrained_backbone=False) load_weights(net, '../timm-efficientdet-pytorch/efficientdet_d5-ef44aea8.pth') config.num_classes = 1 config.image_size = our_image_size net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01)) model_train = DetBenchTrain(net, config) model_eval = DetBenchEval(net, config) manager = ModelManager(model_train, model_eval, device) pretrained_weights_file = 'pretrained.pth' images_val = train_images_df.loc[(train_images_df[fold_column] == fold) & with_boxes_filter, image_id_column].values images_train = train_images_df.loc[(train_images_df[fold_column] != fold), image_id_column].values #images_train = list(images_train) + list(negative_images) + list(spike_images) images_train = list(images_train) + list(negative_images) print(len(images_train), len(images_val)) train_dataset = WheatDataset(images_train, DIR_TRAIN, train_boxes_df, transforms=get_train_transform(), is_test=False) valid_dataset = WheatDataset(images_val, DIR_TRAIN, train_boxes_df, transforms=get_valid_transform(), is_test=True) train_data_loader = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=num_workers, collate_fn=collate_fn, drop_last=True) valid_data_loader = DataLoader(valid_dataset, batch_size=inf_batch_size, shuffle=False, num_workers=num_workers, collate_fn=collate_fn) weights_file = f'{experiment_name}.pth' if os.path.exists(pretrained_weights_file): # continue training print('Continue training, loading weights: ' + pretrained_weights_file) load_weights(net, pretrained_weights_file) manager.run_train(train_data_loader, valid_data_loader, n_epoches=n_epochs, weights_file=weights_file, factor=factor, start_lr=start_lr, min_lr=min_lr, lr_patience=lr_patience, overall_patience=overall_patience, loss_delta=loss_delta) # add tags neptune.log_text('save checkpoints as', weights_file[:-4]) neptune.stop()
'start_lr': start_lr, 'min_lr': min_lr, 'lr_patience': lr_patience, 'overall_patience': overall_patience, 'loss_delta': loss_delta, 'experiment_tag': experiment_tag, } # Create experiment with defined parameters neptune.create_experiment( name=model_name, params=PARAMS, tags=[experiment_name, experiment_tag, f'fold_{fold}'], upload_source_files=['model29.py', 'src/datasets/get_transforms.py']) neptune.append_tags(['grad_accum']) train_boxes_df = pd.read_csv(os.path.join(DATA_DIR, 'fixed_train.csv')) train_images_df = pd.read_csv(os.path.join(DATA_DIR, 'orig_alex_folds.csv')) train_images_df = pd.read_csv( os.path.join(DATA_DIR, 'alex_folds_with_negative.csv')) train_boxes_df = preprocess_boxes(train_boxes_df) # filter tiny boxes as well #train_boxes_df = filter_box_size(train_boxes_df, min_size = 10) #train_boxes_df['area'] = train_boxes_df['w'] * train_boxes_df['h'] train_boxes_df = pd.read_csv(os.path.join(DATA_DIR, 'fixed_train.csv')) print('boxes original: ', len(train_boxes_df)) #spike_df = pd.read_csv(os.path.join(DATA_DIR, 'spike_train.csv')) #train_boxes_df = pd.concat((train_boxes_df, spike_df), axis=0, sort=False)
# Initialize Neptune import neptune neptune.init(api_token="ANONYMOUS", project_qualified_name="shared/colab-test-run") # Basic Example neptune.create_experiment(name='basic-colab-example', params={'learning_rate': 0.1}) neptune.log_metric('accuracy', 0.93) neptune.append_tags(['basic', 'finished_successfully']) # tests exp = neptune.get_experiment() neptune.stop() # tests if set(exp.get_logs().keys()) != set(['accuracy']): raise ValueError() # Keras classification example [Advanced] get_ipython().system(' pip install tensorflow==2.3.1 scikit-plot==0.3.7')
'n_epochs': n_epochs, 'factor': factor, 'start_lr': start_lr, 'min_lr': min_lr, 'lr_patience': lr_patience, 'overall_patience': overall_patience, 'loss_delta': loss_delta, } # Create experiment with defined parameters neptune.create_experiment(name=model_name, params=PARAMS, tags=[experiment_name, experiment_tag], upload_source_files=['train_effdet_val.py']) neptune.append_tags(f'fold_{fold}') train_boxes_df = pd.read_csv(os.path.join(DATA_DIR, 'fixed_train.csv')) train_images_df = pd.read_csv(os.path.join(DATA_DIR, 'orig_alex_folds.csv')) train_boxes_df['x'] = -1 train_boxes_df['y'] = -1 train_boxes_df['w'] = -1 train_boxes_df['h'] = -1 def expand_bbox(x): r = np.array(re.findall("([0-9]+[.]?[0-9]*)", x)) if len(r) == 0: r = [-1, -1, -1, -1] return r
'lr_patience': lr_patience, 'overall_patience': overall_patience, 'loss_delta': loss_delta, 'augs': 'd4' } # Create experiment with defined parameters neptune.create_experiment( name=model_name, params=PARAMS, tags=[experiment_name, experiment_tag, f'fold_{fold}'], upload_source_files=[ 'train_effdet_val2.py', 'src/datasets/get_transforms.py' ]) neptune.append_tags(['augs_d4']) train_boxes_df = pd.read_csv(os.path.join(DATA_DIR, 'fixed_train.csv')) train_images_df = pd.read_csv(os.path.join(DATA_DIR, 'orig_alex_folds.csv')) train_boxes_df = preprocess_boxes(train_boxes_df) # filter tiny boxes as well #train_boxes_df = filter_box_size(train_boxes_df, min_size = 10) def train_box_callback(image_id): records = train_boxes_df[train_boxes_df['image_id'] == image_id] return records[['x', 'y', 'w', 'h']].values def split_prediction_string(str):
def do_main(): neptune.init('ods/wheat') # Create experiment with defined parameters neptune.create_experiment(name=model_name, params=PARAMS, tags=[experiment_name, experiment_tag], upload_source_files=[os.path.basename(__file__)]) neptune.append_tags(f'fold_{fold}') neptune.append_tags(['grad_accum']) device = torch.device(f'cuda:{gpu_number}') if torch.cuda.is_available( ) else torch.device('cpu') print(device) print(len(train_boxes_df)) print(len(train_images_df)) # Leave only > 0 print('Leave only train images with boxes (validation)') with_boxes_filter = train_images_df[image_id_column].isin( train_boxes_df[image_id_column].unique()) # config models fro train and validation config = get_efficientdet_config('tf_efficientdet_d5') net = EfficientDet(config, pretrained_backbone=False) load_weights(net, '../timm-efficientdet-pytorch/efficientdet_d5-ef44aea8.pth') config.num_classes = 1 config.image_size = our_image_size net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01)) model_train = DetBenchTrain(net, config) model_eval = DetBenchEval(net, config) manager = ModelManager(model_train, model_eval, device) images_val = train_images_df.loc[(train_images_df[fold_column] == fold) & with_boxes_filter, image_id_column].values images_train = train_images_df.loc[(train_images_df[fold_column] != fold) & with_boxes_filter, image_id_column].values print( f'\nTrain images:{len(images_train)}, validation images {len(images_val)}' ) # get augs #augs_dict = set_augmentations(our_image_size) # get datasets train_dataset = WheatDataset( image_ids=images_train[:160], image_dir=DIR_TRAIN, boxes_df=train_boxes_df, transforms=get_train_transform(our_image_size), is_test=False) valid_dataset = WheatDataset( image_ids=images_val[:160], image_dir=DIR_TRAIN, boxes_df=train_boxes_df, transforms=get_valid_transform(our_image_size), is_test=True) train_data_loader = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=num_workers, collate_fn=collate_fn, drop_last=True) valid_data_loader = DataLoader(valid_dataset, batch_size=inf_batch_size, shuffle=False, num_workers=num_workers, collate_fn=collate_fn) weights_file = f'../checkpoints/{model_name}/{experiment_name}.pth' #pretrain_weights_file = f'{checkpoints_dir}/{experiment_name}.pth' #if os.path.exists(pretrain_weights_file): # print(f'Continue training, loading weights from {pretrain_weights_file}') # load_weights(net, pretrain_weights_file) manager.run_train(train_generator=train_data_loader, val_generator=valid_data_loader, n_epoches=n_epochs, weights_file=weights_file, factor=factor, start_lr=start_lr, min_lr=min_lr, lr_patience=lr_patience, overall_patience=overall_patience, loss_delta=loss_delta) # add tags neptune.log_text('save checkpoints as', weights_file[:-4]) neptune.stop()