def initialize_cfg_and_register(trial_id,sample,output_dir = None,): ''' setup base configuration of model SEE MORE AT https://detectron2.readthedocs.io/en/latest/modules/config.html ''' if sample[1]: year_str = "w20_" else: year_str = "" data_name = f'filet_{year_str}{sample[0]}_train' try: DatasetCatalog.register(data_name,lambda : get_data_dicts(data_dir,'train',sample[2])) MetadataCatalog.get(data_name).set(thing_classes = ['filet']) except AssertionError as e: print(e) cfg = get_cfg() cfg.merge_from_file(model_zoo.get_config_file(f'{model_name}.yaml')) cfg.DATASETS.TRAIN = (data_name,) cfg.DATASETS.TEST = ('filet_val',) # Use this with trainer_cls : TrainerPeriodicEval if you want to do validation after every iterations cfg.TEST.EVAL_PERIOD = 500 cfg.DATALOADER.NUM_WORKERS = 6 #add more workerss until it gives warnings. cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(f'{model_name}.yaml') cfg.SOLVER.IMS_PER_BATCH = 3 #maybe more? if output_dir is None: cfg.OUTPUT_DIR = base_output_dir else: cfg.OUTPUT_DIR = output_dir cfg.SOLVER.STEPS =(5000,25000) cfg.SOLVER.GAMMA = 0.5 os.makedirs(cfg.OUTPUT_DIR,exist_ok=True) cfg.SOLVER.MAX_ITER = 10**5 cfg.SOLVER.BASE_LR = 0.00025 cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128 #(default: 512) cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1 return cfg
def test_register_data(self): COCO_dicts = {split: get_data_dicts(data_dir,split) for split in splits } register_data('filet',['train','val'],COCO_dicts)
def test_get_data_dicts(self): for split in splits: data = get_data_dicts(data_dir, split)
if ma: if ma.group()=='2020-': data_pairs_2020[front] = files elif ma.group()=='2021-': data_pairs_2021[front] = files else: print("warning: unknown matched year") print(front,ma.group()) else: print("warning: unknown year") print(front) return data_pairs_2020,data_pairs_2021 data_pairs_2020 , data_pairs_2021 = partition_pairs_by_year(train_pairs) _ , data_pairs_val = partition_pairs_by_year(val_pairs) DatasetCatalog.register('filet_val', lambda : get_data_dicts(data_dir, 'val', data_pairs_val)) MetadataCatalog.get('filet_val').set(thing_classes=['filet']) def generate_train_sets(include2020 = True): if include2020: sample_space = train_pairs else: sample_space = data_pairs_2021 data_pair_train_samples= [] for expo in range(6,floor(log(len(sample_space),2))+1): data_keys_sample = choice(list(sample_space.keys()),2**expo,replace=False) data_pair_train_samples.append((2**expo,include2020, {key : sample_space[key] for key in data_keys_sample })) if (len(sample_space) - floor(log(len(sample_space),2)) )/len(sample_space)>0.1: #if there is significant difference in taking total set, then take also total set data_pair_train_samples.append((len(sample_space),include2020,sample_space)) return data_pair_train_samples
from numpy.random import choice, randint, uniform from trainers import TI_Trainer from detectron2.config import get_cfg from detectron2 import model_zoo from detectron2.data import DatasetMapper, build_detection_train_loader import detectron2.data.transforms as T from detectron2.evaluation import COCOEvaluator from pruners import SHA from trainers import TrainerPeriodicEval from hyperoptimization import D2_hyperopt_Base from numpy import random from data_utils import get_data_dicts, register_data splits = ['train', 'val'] data_dir = "/pers_files/test_set" COCO_dicts = {split: get_data_dicts(data_dir, split) for split in splits } #converting TI-annotation of pictures to COCO annotations. data_names = register_data( 'filet', ['train', 'val'], COCO_dicts, {'thing_classes': ['filet']}) #register data by str name in D2 api output_dir = f'{data_dir}/output' print(data_names) def initialize_base_cfg(model_name, cfg=None): ''' setup base configuration of model SEE MORE AT https://detectron2.readthedocs.io/en/latest/modules/config.html ''' if cfg is None: cfg = get_cfg()