def initialize_cfg_and_register(trial_id,sample,output_dir = None,):
'''
setup base configuration of model SEE MORE AT https://detectron2.readthedocs.io/en/latest/modules/config.html
'''
if sample[1]:
year_str = "w20_"
else:
year_str = ""
data_name = f'filet_{year_str}{sample[0]}_train'
try:
DatasetCatalog.register(data_name,lambda : get_data_dicts(data_dir,'train',sample[2]))
MetadataCatalog.get(data_name).set(thing_classes = ['filet'])
except AssertionError as e:
print(e)
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file(f'{model_name}.yaml'))
cfg.DATASETS.TRAIN = (data_name,)
cfg.DATASETS.TEST = ('filet_val',) # Use this with trainer_cls : TrainerPeriodicEval if you want to do validation after every iterations
cfg.TEST.EVAL_PERIOD = 500
cfg.DATALOADER.NUM_WORKERS = 6 #add more workerss until it gives warnings.
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(f'{model_name}.yaml')
cfg.SOLVER.IMS_PER_BATCH = 3 #maybe more?
if output_dir is None:
cfg.OUTPUT_DIR = base_output_dir
else:
cfg.OUTPUT_DIR = output_dir
cfg.SOLVER.STEPS =(5000,25000)
cfg.SOLVER.GAMMA = 0.5
os.makedirs(cfg.OUTPUT_DIR,exist_ok=True)
cfg.SOLVER.MAX_ITER = 10**5
cfg.SOLVER.BASE_LR = 0.00025
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 128 #(default: 512)
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
return cfg
def test_register_data(self):
COCO_dicts = {split: get_data_dicts(data_dir,split) for split in splits }
register_data('filet',['train','val'],COCO_dicts)
def test_get_data_dicts(self):
for split in splits:
data = get_data_dicts(data_dir, split)
if ma:
if ma.group()=='2020-':
data_pairs_2020[front] = files
elif ma.group()=='2021-':
data_pairs_2021[front] = files
else:
print("warning: unknown matched year")
print(front,ma.group())
else:
print("warning: unknown year")
print(front)
return data_pairs_2020,data_pairs_2021
data_pairs_2020 , data_pairs_2021 = partition_pairs_by_year(train_pairs)
_ , data_pairs_val = partition_pairs_by_year(val_pairs)
DatasetCatalog.register('filet_val', lambda : get_data_dicts(data_dir, 'val', data_pairs_val))
MetadataCatalog.get('filet_val').set(thing_classes=['filet'])
def generate_train_sets(include2020 = True):
if include2020:
sample_space = train_pairs
else:
sample_space = data_pairs_2021
data_pair_train_samples= []
for expo in range(6,floor(log(len(sample_space),2))+1):
data_keys_sample = choice(list(sample_space.keys()),2**expo,replace=False)
data_pair_train_samples.append((2**expo,include2020, {key : sample_space[key] for key in data_keys_sample }))
if (len(sample_space) - floor(log(len(sample_space),2)) )/len(sample_space)>0.1: #if there is significant difference in taking total set, then take also total set
data_pair_train_samples.append((len(sample_space),include2020,sample_space))
return data_pair_train_samples
from numpy.random import choice, randint, uniform
from trainers import TI_Trainer
from detectron2.config import get_cfg
from detectron2 import model_zoo
from detectron2.data import DatasetMapper, build_detection_train_loader
import detectron2.data.transforms as T
from detectron2.evaluation import COCOEvaluator
from pruners import SHA
from trainers import TrainerPeriodicEval
from hyperoptimization import D2_hyperopt_Base
from numpy import random
from data_utils import get_data_dicts, register_data
splits = ['train', 'val']
data_dir = "/pers_files/test_set"
COCO_dicts = {split: get_data_dicts(data_dir, split)
for split in splits
} #converting TI-annotation of pictures to COCO annotations.
data_names = register_data(
'filet', ['train', 'val'], COCO_dicts,
{'thing_classes': ['filet']}) #register data by str name in D2 api
output_dir = f'{data_dir}/output'
print(data_names)
def initialize_base_cfg(model_name, cfg=None):
'''
setup base configuration of model SEE MORE AT https://detectron2.readthedocs.io/en/latest/modules/config.html
'''
if cfg is None:
cfg = get_cfg()