def parse_aide_config(cls, config, detectron2cfg=None): if detectron2cfg is None: detectron2cfg = get_cfg() if isinstance(config, dict): for key in config.keys(): if isinstance(key, str) and key.startswith('DETECTRON2.'): value = optionsHelper.get_hierarchical_value( config[key], ['value', 'id']) if isinstance(value, list): for v in range(len(value)): value[v] = optionsHelper.get_hierarchical_value( value[v], ['value', 'id']) # copy over to Detectron2 configuration tokens = key.split('.') attr = detectron2cfg for t in range(1, len(tokens)): # skip "DETECTRON2" marker if t == len(tokens) - 1: # last element setattr(attr, tokens[t], value) else: attr = getattr(attr, tokens[t]) else: GenericDetectron2Model.parse_aide_config( config[key], detectron2cfg) return detectron2cfg
def initializeModel(self, stateDict, data, addMissingLabelClasses=False, removeObsoleteLabelClasses=False): ''' Converts the provided stateDict from a bytes array to a torch-loadable object and initializes a model from it. Also returns a 'labelClassMap', defining the indexing between label classes and the model. If the stateDict object is None, a new model and labelClassMap are crea- ted from the defaults. If "addMissingLabelClasses" is True, new output neurons are appended for label classes that are not present in the model's current labelclass map, and the map is updated. Likewise, if "removeObsoleteLabelClasses" is True, existing outputs for label classes that are not present in the set of label classes anymore are removed, and the map is also updated. ''' # initialize model if stateDict is not None: stateDict = torch.load(io.BytesIO(stateDict), map_location=lambda storage, loc: storage) model = self.model_class.loadFromStateDict(stateDict) # mapping labelclass (UUID) to index in model (number) labelclassMap = stateDict['labelclassMap'] if addMissingLabelClasses or removeObsoleteLabelClasses: # modification of model outputs if hasattr(model, 'updateModel'): model.updateModel(data['labelClasses'], addMissingLabelClasses, removeObsoleteLabelClasses) else: # create new label class map labelclassMap = {} for idx, lcID in enumerate(data['labelClasses']): labelclassMap[ lcID] = idx #NOTE: we do not use the labelclass' serial 'idx', since this might contain gaps # self.options['options']['model']['labelclassMap'] = labelclassMap #TODO: never used and obsolete with new options format # initialize a fresh model modelKwargs = {} modelOptions = optionsHelper.get_hierarchical_value( self.options, ['options', 'model'], []) for key in modelOptions.keys(): if key not in optionsHelper.RESERVED_KEYWORDS: modelKwargs[key] = optionsHelper.get_hierarchical_value( modelOptions[key], ['value', 'id']) modelKwargs['labelclassMap'] = labelclassMap model = self.model_class(**modelKwargs) return model, labelclassMap
def _get_config(self): cfg = get_cfg() cfg.set_new_allowed(True) # augment and initialize Detectron2 cfg with selected model defaultConfig = optionsHelper.get_hierarchical_value( self.options, ['options', 'model', 'config', 'value', 'id']) if isinstance(defaultConfig, str): # try to load from Detectron2's model zoo try: configFile = model_zoo.get_config_file(defaultConfig) except: # not available; try to load locally instead configFile = os.path.join( os.getcwd(), 'ai/models/detectron2/_functional/configs', defaultConfig) if not os.path.exists(configFile): configFile = None if configFile is not None: cfg.merge_from_file(configFile) try: cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(defaultConfig) except: pass return cfg
def _parse_transform(tr): tr_kwargs = {} if isinstance(tr, dict): tr_class = optionsHelper.get_hierarchical_value(tr, ['id']) for key in tr.keys(): if key not in optionsHelper.RESERVED_KEYWORDS: tr_kwargs[key] = optionsHelper.get_hierarchical_value( tr[key], ['value']) elif isinstance(tr, str): tr_class = tr if tr_class.endswith('DefaultTransform'): subTr = GenericPyTorchModel.parseTransforms(tr['transform']) tr_kwargs = {'transform': subTr} elif tr_class == 'torchvision.transforms.Normalize': mean = optionsHelper.get_hierarchical_value( tr_kwargs, ['mean', 'value']) std = optionsHelper.get_hierarchical_value( tr_kwargs, ['std', 'value']) tr_kwargs = { 'mean': [mean[0]['value'], mean[1]['value'], mean[2]['value']], 'std': [std[0]['value'], std[1]['value'], std[2]['value']] } elif tr_class == 'torchvision.transforms.ColorJitter': brightness = optionsHelper.get_hierarchical_value( tr_kwargs, ['brightness', 'value']) contrast = optionsHelper.get_hierarchical_value( tr_kwargs, ['contrast', 'value']) saturation = optionsHelper.get_hierarchical_value( tr_kwargs, ['saturation', 'value']) hue = optionsHelper.get_hierarchical_value( tr_kwargs, ['hue', 'value']) tr_kwargs = { 'brightness': [ brightness[0]['value'], brightness[1]['value'], ], 'contrast': [ contrast[0]['value'], contrast[1]['value'], ], 'saturation': [ saturation[0]['value'], saturation[1]['value'], ], 'hue': [ hue[0]['value'], hue[1]['value'], ] } elif tr_class.endswith('Compose'): sub_transforms = GenericPyTorchModel.parseTransforms( tr['transforms']) tr_kwargs = {'transforms': sub_transforms} #TODO: others? return parse_transforms({'class': tr_class, 'kwargs': tr_kwargs})
def _get_config(self): cfg = get_cfg() add_torchvision_classifier_config(cfg) defaultConfig = optionsHelper.get_hierarchical_value( self.options, ['defs', 'model']) if isinstance(defaultConfig, dict): defaultConfig = defaultConfig['id'] configFile = os.path.join(os.getcwd(), 'ai/models/detectron2/_functional/configs', defaultConfig) cfg.merge_from_file(configFile) return cfg
def __init__(self, project, config, dbConnector, fileServer, options): super(GenericPyTorchModel, self).__init__(project, config, dbConnector, fileServer, options) # try to fill and substitute global definitions in JSON-enhanced options if isinstance(options, dict) and 'defs' in options: try: updatedOptions = optionsHelper.substitute_definitions( options.copy()) self.options = updatedOptions except: # something went wrong; ignore pass # retrieve executables try: self.model_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'model', 'class'])) except: self.model_class = None try: self.criterion_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'criterion', 'class'])) except: self.criterion_class = None try: self.optim_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'optim'])) except: self.optim_class = SGD try: self.dataset_class = get_class_executable( optionsHelper.get_hierarchical_value(self.options, ['options', 'dataset'])) except: self.dataset_class = None
def _get_config(self): cfg = get_cfg() add_deeplab_config(cfg) defaultConfig = optionsHelper.get_hierarchical_value(self.options, ['options', 'model', 'config', 'value', 'id']) configFile = os.path.join(os.getcwd(), 'ai/models/detectron2/_functional/configs', defaultConfig) cfg.merge_from_file(configFile) # disable SyncBatchNorm if not running on distributed system if comm.get_world_size() <= 1: cfg.MODEL.RESNETS.NORM = 'BN' cfg.MODEL.SEM_SEG_HEAD.NORM = 'BN' return cfg
def initializeTransforms(self, mode='train', options=None): ''' AIDE's Detectron2-compliant models all support the same transforms and thus can be initialized the same way. "mode" determines whether the transforms specified for 'train' or for 'inference' are to be initialized. If "options" contains a dict of AIDE model options, the transforms are to be initialized from there; otherwise the current class-specific ones are used. ''' assert mode in ('train', 'inference'), 'Invalid transform mode specified' if isinstance(options, dict): opt = copy.deepcopy(options) else: opt = copy.deepcopy(self.options) opt = optionsHelper.substitute_definitions(opt) # parse transforms transforms = [] #TODO: add a resize transform in any case transformOpts = optionsHelper.get_hierarchical_value( opt, ['options', mode, 'transform', 'value']) for tr in transformOpts: trClass = tr['id'] args = optionsHelper.filter_reserved_children(tr, True) for a in args.keys(): args[a] = optionsHelper.get_hierarchical_value( args[a], ['value']) if a == 'interp': #TODO: ugly solution to convert interpolation methods args[a] = int(args[a]) # initialize transform = getattr(T, trClass)(**args) transforms.append(transform) return transforms
def verifyOptions(cls, options): defaultOptions = cls.getDefaultOptions() if options is None: return {'valid': True, 'options': defaultOptions} try: if isinstance(options, str): options = json.loads(options) options = optionsHelper.substitute_definitions(options) except Exception as e: return { 'valid': False, 'errors': [f'Options are not in a proper format (message: {str(e)}).'] } try: # mandatory field: model config modelConfig = optionsHelper.get_hierarchical_value( options, ['options', 'model', 'config', 'value']) if modelConfig is None: raise Exception('missing model type field in options.') opts, warnings, errors = optionsHelper.verify_options( options['options'], autoCorrect=True) options['options'] = opts return { 'valid': not len(errors), 'warnings': warnings, 'errors': errors, 'options': options } except Exception as e: return { 'valid': False, 'errors': [ f'An error occurred trying to verify options (message: {str(e)}).' ] }
def train(self, stateDict, data, updateStateFun): ''' Main training function. ''' # initialize model model, stateDict, _, projectToStateMap = self.initializeModel( stateDict, data, False) # wrap dataset for usage with Detectron2 ignoreUnsure = optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'ignore_unsure', 'value'], fallback=True) transforms = self.initializeTransforms(mode='train') indexMap = self._get_labelclass_index_map(stateDict['labelclassMap'], False) try: imageFormat = self.detectron2cfg.INPUT.FORMAT assert imageFormat.upper() in ('RGB', 'BGR') except: imageFormat = 'BGR' datasetMapper = Detectron2DatasetMapper(self.project, self.fileServer, transforms, True, imageFormat, classIndexMap=indexMap) dataLoader = build_detection_train_loader( dataset=getDetectron2Data( data, stateDict['labelclassMap'], projectToStateMap, ignoreUnsure, self.detectron2cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS), mapper=datasetMapper, total_batch_size=self.detectron2cfg.SOLVER.IMS_PER_BATCH * comm.get_world_size(), #TODO: verify aspect_ratio_grouping=True, num_workers=0) numImg = len(data['images']) # train model.train() optimizer = self._build_optimizer(self.detectron2cfg, model) scheduler = self._build_lr_scheduler(self.detectron2cfg, optimizer) imgCount = 0 start_iter = 0 #TODO tbar = trange(numImg) dataLoaderIter = iter(dataLoader) with EventStorage(start_iter) as storage: for idx in range(numImg): batch = next(dataLoaderIter) storage.iter = idx #TODO: start_iter loss_dict = model(batch) losses = sum(loss_dict.values()) assert torch.isfinite(losses).all(), \ 'Model produced Inf and/or NaN values; training was aborted. Try reducing the learning rate.' loss_dict_reduced = { k: v.item() for k, v in comm.reduce_dict(loss_dict).items() } losses_reduced = sum(loss for loss in loss_dict_reduced.values()) if comm.is_main_process(): storage.put_scalars(total_loss=losses_reduced, **loss_dict_reduced) optimizer.zero_grad() losses.backward() optimizer.step() scheduler.step() # update worker state tbar.update(1) imgCount += len(batch) updateStateFun(state='PROGRESS', message='training', done=imgCount, total=numImg) stats = storage.latest() for key in stats: if isinstance(stats[key], tuple): stats[key] = stats[key][0] tbar.close() # all done; return state dict as bytes and stats return self.exportModelState(stateDict, model), stats
def initializeModel(self, stateDict, data, revertProjectToStateMap=False): ''' Loads Bytes object "stateDict" through torch and looks for a Detectron2 config to initialize the model structure, optionally with pre-trained weights if available. Returns the model instance, the state dict, inter alia augmented with a 'labelClassMap', defining the indexing between label classes and the model, and a list of new label classes (i.e., those that are present in the "data" dict, but not in the model definition). If the stateDict object is None, a new model and labelClassMap are crea- ted from the defaults. Note that this function does NOT modify the model w.r.t. the actually present label classes in the project. This functionality requires re- configuring the model's prediction output and depends on the model imple- mentation. ''' # check if user has forced creating a new model forceNewModel = optionsHelper.get_hierarchical_value( self.options, ['options', 'model', 'force', 'value'], fallback=False) if isinstance(forceNewModel, bool) and forceNewModel: # print warning and reset flag print( f'[{self.project}] User has selected to force recreating a brand-new model.' ) optionsHelper.set_hierarchical_value( self.options, ['options', 'model', 'force', 'value'], False) else: forceNewModel = False # load state dict if stateDict is not None and not forceNewModel: if not isinstance(stateDict, dict): stateDict = torch.load( io.BytesIO(stateDict), map_location=lambda storage, loc: storage) else: stateDict = {} # retrieve Detectron2 cfg if 'detectron2cfg' in stateDict and not forceNewModel: # medium priority: model overrides self.detectron2cfg.set_new_allowed(True) self.detectron2cfg.update(stateDict['detectron2cfg']) # top priority: AIDE config overrides self.detectron2cfg = self.parse_aide_config(self.options, self.detectron2cfg) stateDict['detectron2cfg'] = self.detectron2cfg # check if CUDA is available; set to CPU temporarily if not if not torch.cuda.is_available(): self.detectron2cfg.MODEL.DEVICE = 'cpu' # construct model and load state model = detectron2.modeling.build_model(self.detectron2cfg) checkpointer = DetectionCheckpointerInMem(model) if 'model' in stateDict and not forceNewModel: # trained weights available checkpointer.loadFromObject(stateDict) else: # fresh model; initialize from Detectron2 weights checkpointer.load(self.detectron2cfg.MODEL.WEIGHTS) # load or create labelclass map if 'labelclassMap' in stateDict and not forceNewModel: labelclassMap = stateDict['labelclassMap'] else: labelclassMap = {} # add existing label classes; any non-UUID entry will be discarded during prediction try: pretrainedDataset = self.detectron2cfg.DATASETS.TRAIN if isinstance(pretrainedDataset, list) or isinstance( pretrainedDataset, tuple): pretrainedDataset = pretrainedDataset[0] pretrainedMeta = MetadataCatalog.get(pretrainedDataset) for idx, cID in enumerate(pretrainedMeta.thing_classes): labelclassMap[cID] = idx except: pass # check data for new label classes projectToStateMap = {} newClasses = [] for lcID in data['labelClasses']: if lcID not in labelclassMap: # check if original label class got re-mapped lcMap_origin_id = data['labelClasses'][lcID].get( 'labelclass_id_model', None) if lcMap_origin_id is None or lcMap_origin_id not in labelclassMap: # no remapping done; class really is new newClasses.append(lcID) else: # class has been re-mapped if revertProjectToStateMap: projectToStateMap[lcMap_origin_id] = lcID else: projectToStateMap[lcID] = lcMap_origin_id stateDict['labelclassMap'] = labelclassMap # parallelize model (if architecture supports it) #TODO: try out whether/how well this works distributed = comm.get_world_size() > 1 if distributed: model = DistributedDataParallel(model, device_ids=[comm.get_local_rank()], broadcast_buffers=False) return model, stateDict, newClasses, projectToStateMap
def get_device(self): device = optionsHelper.get_hierarchical_value( self.options, ['options', 'general', 'device', 'value', 'id']) if 'cuda' in device and not torch.cuda.is_available(): device = 'cpu' return device
def _convertOldOptions(options, defaults): ''' Receives options in the previous JSON encoding and converts them to the new GUI-enhanced scheme. Returns the new, converted options accordingly. ''' newOptions = defaults.copy() warnings = [] # update defaults key by key for key in OPTIONS_MAPPING.keys(): newTokens = ['options'] newTokens.extend(key.split('.')) oldTokens = OPTIONS_MAPPING[key].split('.') oldValue = optionsHelper.get_hierarchical_value( options, oldTokens, None) if oldValue is None: warnings.append( f'Value for options "{key}" could not be found in given options (expected location: "{OPTIONS_MAPPING[key]}").' ) else: optionsHelper.set_hierarchical_value(newOptions, newTokens, oldValue) # take special care of the optimizer: try all possible values (only the ones present will be retained) currentOptimType = options['train']['optim']['class'] optionsHelper.set_hierarchical_value(newOptions, ('train', 'optim', 'value'), currentOptimType) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'lr', 'value'), ('train', 'optim', 'kwargs', 'lr')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'weight_decay', 'value'), ('train', 'optim', 'kwargs', 'weight_decay')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'momentum', 'value'), ('train', 'optim', 'kwargs', 'momentum')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'alpha', 'value'), ('train', 'optim', 'kwargs', 'alpha')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'centered', 'value'), ('train', 'optim', 'kwargs', 'centered')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'dampening', 'value'), ('train', 'optim', 'kwargs', 'dampening')) optionsHelper.update_hierarchical_value( options, newOptions, ('train', 'optim', 'options', currentOptimType, 'nesterov', 'value'), ('train', 'optim', 'kwargs', 'nesterov')) # also take special care of the transforms def _update_transforms(currentTransforms): newTransforms = [] for tr in currentTr_train: # get from template definition and then replace values trClass = tr['class'] if trClass not in newOptions['defs']['transform']: warnings.append( f'Transform "{trClass}" is not defined in the new scheme and will be substituted appropriately.' ) continue newTr = newOptions['defs']['transform'][trClass] for kw in tr['kwargs'].keys(): if kw == 'size': newTr['width']['value'] = tr['kwargs']['size'][0] newTr['height']['value'] = tr['kwargs']['size'][1] elif kw in ('brightness', 'contrast', 'saturation', 'hue'): newTr[kw]['minV']['value'] = 0 newTr[kw]['maxV']['value'] = tr['kwargs'][kw] warnings.append( f'{kw} values of transforms have been set as maximums (min: 0).' ) elif kw in ('mean', 'std'): newTr['mean']['r'] = tr['kwargs'][kw][0] newTr['mean']['g'] = tr['kwargs'][kw][1] newTr['mean']['b'] = tr['kwargs'][kw][2] elif kw in newTr: newTr[kw]['value'] = tr['kwargs'][kw] newTransforms.append(newTr) return newTransforms currentTr_train = options['train']['transform']['kwargs']['transforms'] newTr_train = _update_transforms(currentTr_train) newOptions['options']['train']['transform']['value'] = newTr_train currentTr_inference = options['inference']['transform']['kwargs'][ 'transforms'] newTr_inference = _update_transforms(currentTr_inference) newOptions['options']['inference']['transform'][ 'value'] = newTr_inference print('Old RetinaNet options successfully converted to new format.') return newOptions, warnings
def inference(self, stateDict, data, updateStateFun): # initialize model if stateDict is None: raise Exception( 'No trained model state found, but required for inference.') # read state dict from bytes model, labelclassMap = self.initializeModel(stateDict, data) # initialize data loader, dataset, transforms inputSize = (int( optionsHelper.get_hierarchical_value( self.options, ['options', 'general', 'imageSize', 'width', 'value'])), int( optionsHelper.get_hierarchical_value( self.options, [ 'options', 'general', 'imageSize', 'height', 'value' ]))) transform = RetinaNet._init_transform_instances( optionsHelper.get_hierarchical_value( self.options, ['options', 'inference', 'transform', 'value']), inputSize) dataset = BoundingBoxesDataset(data=data, fileServer=self.fileServer, labelclassMap=labelclassMap, transform=transform) dataEncoder = encoder.DataEncoder( minIoU_pos=0.5, maxIoU_neg=0.4) # IoUs don't matter for inference collator = collation.Collator(self.project, self.dbConnector, ( inputSize[1], inputSize[0], ), dataEncoder) dataLoader = DataLoader(dataset=dataset, collate_fn=collator.collate_fn, shuffle=False) # perform inference response = {} device = self.get_device() model.to(device) imgCount = 0 for (img, _, _, fVec, imgID) in tqdm(dataLoader): # TODO: implement feature vectors # if img is not None: # dataItem = img.to(device) # isFeatureVector = False # else: # dataItem = fVec.to(device) # isFeatureVector = True dataItem = img.to(device) with torch.no_grad(): bboxes_pred_batch, labels_pred_batch = model( dataItem, False) #TODO: isFeatureVector bboxes_pred_batch, labels_pred_batch, confs_pred_batch = dataEncoder.decode( bboxes_pred_batch.squeeze(0).cpu(), labels_pred_batch.squeeze(0).cpu(), inputSize, cls_thresh=optionsHelper.get_hierarchical_value( self.options, [ 'options', 'inference', 'encoding', 'cls_thresh', 'value' ], fallback=0.1), nms_thresh=optionsHelper.get_hierarchical_value( self.options, [ 'options', 'inference', 'encoding', 'nms_thresh', 'value' ], fallback=0.1), numPred_max=int( optionsHelper.get_hierarchical_value(self.options, [ 'options', 'inference', 'encoding', 'numPred_max', 'value' ], fallback=128)), return_conf=True) for i in range(len(imgID)): bboxes_pred = bboxes_pred_batch[i] labels_pred = labels_pred_batch[i] confs_pred = confs_pred_batch[i] if bboxes_pred.dim() == 2: bboxes_pred = bboxes_pred.unsqueeze(0) labels_pred = labels_pred.unsqueeze(0) confs_pred = confs_pred.unsqueeze(0) # convert bounding boxes to YOLO format predictions = [] bboxes_pred_img = bboxes_pred[0, ...] labels_pred_img = labels_pred[0, ...] confs_pred_img = confs_pred[0, ...] if len(bboxes_pred_img): bboxes_pred_img[:, 2] -= bboxes_pred_img[:, 0] bboxes_pred_img[:, 3] -= bboxes_pred_img[:, 1] bboxes_pred_img[:, 0] += bboxes_pred_img[:, 2] / 2 bboxes_pred_img[:, 1] += bboxes_pred_img[:, 3] / 2 bboxes_pred_img[:, 0] /= inputSize[0] bboxes_pred_img[:, 1] /= inputSize[1] bboxes_pred_img[:, 2] /= inputSize[0] bboxes_pred_img[:, 3] /= inputSize[1] # limit to image bounds bboxes_pred_img = torch.clamp(bboxes_pred_img, 0, 1) # append to dict for b in range(bboxes_pred_img.size(0)): bbox = bboxes_pred_img[b, :] label = labels_pred_img[b] logits = confs_pred_img[b, :] predictions.append({ 'x': bbox[0].item(), 'y': bbox[1].item(), 'width': bbox[2].item(), 'height': bbox[3].item(), 'label': dataset.labelclassMap_inv[label.item()], 'logits': logits.numpy().tolist( ), #TODO: for AL criterion? 'confidence': torch.max(logits).item() }) response[imgID[i]] = { 'predictions': predictions, #TODO: exception if fVec is not torch tensor: 'fVec': io.BytesIO(fVec.numpy().astype(np.float32)).getvalue() } # update worker state imgCount += len(imgID) updateStateFun(state='PROGRESS', message='predicting', done=imgCount, total=len(dataLoader.dataset)) model.cpu() if 'cuda' in device: torch.cuda.empty_cache() return response
def train(self, stateDict, data, updateStateFun): ''' Initializes a model based on the given stateDict and a data loader from the provided data and trains the model, taking into account the parameters speci- fied in the 'options' given to the class. Returns a serializable state dict of the resulting model. ''' # initialize model model, labelclassMap = self.initializeModel( stateDict, data, optionsHelper.get_hierarchical_value( self.options, ['options', 'general', 'labelClasses', 'add_missing', 'value' ]), optionsHelper.get_hierarchical_value(self.options, [ 'options', 'general', 'labelClasses', 'remove_obsolete', 'value' ])) # setup transform, data loader, dataset, optimizer, criterion inputSize = (int( optionsHelper.get_hierarchical_value( self.options, ['options', 'general', 'imageSize', 'width', 'value'])), int( optionsHelper.get_hierarchical_value( self.options, [ 'options', 'general', 'imageSize', 'height', 'value' ]))) transform = RetinaNet._init_transform_instances( optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'transform', 'value']), inputSize) dataset = BoundingBoxesDataset( data=data, fileServer=self.fileServer, labelclassMap=labelclassMap, targetFormat='xyxy', transform=transform, ignoreUnsure=optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'encoding', 'ignore_unsure', 'value'], fallback=False)) dataEncoder = encoder.DataEncoder( minIoU_pos=optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'encoding', 'minIoU_pos', 'value'], fallback=0.5), maxIoU_neg=optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'encoding', 'maxIoU_neg', 'value'], fallback=0.4)) collator = collation.Collator(self.project, self.dbConnector, ( inputSize[1], inputSize[0], ), dataEncoder) dataLoader = DataLoader( dataset=dataset, collate_fn=collator.collate_fn, shuffle=optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'dataLoader', 'shuffle', 'value'], fallback=True)) # optimizer optimArgs = optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'optim', 'value'], None) optimArgs_out = {} optimClass = get_class_executable(optimArgs['id']) for key in optimArgs.keys(): if key not in optionsHelper.RESERVED_KEYWORDS: optimArgs_out[key] = optionsHelper.get_hierarchical_value( optimArgs[key], ['value']) optimizer = optimClass(params=model.parameters(), **optimArgs_out) # loss criterion critArgs = optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'criterion'], None) critArgs_out = {} for key in critArgs.keys(): if key not in optionsHelper.RESERVED_KEYWORDS: critArgs_out[key] = optionsHelper.get_hierarchical_value( critArgs[key], ['value']) criterion = loss.FocalLoss(**critArgs_out) # train model device = self.get_device() seed = int( optionsHelper.get_hierarchical_value( self.options, ['options', 'general', 'seed', 'value'], fallback=0)) torch.manual_seed(seed) if 'cuda' in device: torch.cuda.manual_seed(seed) model.to(device) imgCount = 0 for (img, bboxes_target, labels_target, fVec, _) in tqdm(dataLoader): img, bboxes_target, labels_target = img.to(device), \ bboxes_target.to(device), \ labels_target.to(device) optimizer.zero_grad() bboxes_pred, labels_pred = model(img) loss_value = criterion(bboxes_pred, bboxes_target, labels_pred, labels_target) loss_value.backward() optimizer.step() # check for Inf and NaN values and raise exception if needed if any([ torch.any(torch.isinf(bboxes_pred)).item(), torch.any(torch.isinf(labels_pred)).item(), torch.any(torch.isnan(bboxes_pred)).item(), torch.any(torch.isnan(labels_pred)).item() ]): raise Exception( 'Model produced Inf and/or NaN values; training was aborted. Try reducing the learning rate.' ) # update worker state imgCount += img.size(0) updateStateFun(state='PROGRESS', message='training', done=imgCount, total=len(dataLoader.dataset)) # all done; return state dict as bytes return self.exportModelState(model)
def verifyOptions(options): # get default options to compare to defaultOptions = RetinaNet.getDefaultOptions() # updated options with modifications made if options is None: updatedOptions = defaultOptions.copy() else: if not isinstance(options, dict): try: options = json.loads(options) except Exception as e: return { 'valid': False, 'warnings': [], 'errors': [ f'Options are not in valid JSON format (message: "{str(e)}").' ] } updatedOptions = options.copy() result = {'valid': True, 'warnings': [], 'errors': []} if not 'defs' in updatedOptions: # old version (without GUI formatting): convert first updatedOptions, warnings = RetinaNet._convertOldOptions( updatedOptions, defaultOptions) result['warnings'].append( 'Options have been converted to new format.') result['warnings'].extend(warnings) # flatten and fill globals updatedOptions = optionsHelper.substitute_definitions(updatedOptions) # do the verification missingClassOptions = optionsHelper.get_hierarchical_value( updatedOptions, ['options', 'general', 'labelClasses']) if not isinstance(missingClassOptions, dict): updatedOptions['options']['general']['labelClasses'] = \ optionsHelper.get_hierarchical_value(defaultOptions, ['options', 'general', 'labelClasses']) #TODO: verify rest # verify transforms transforms_train = updatedOptions['options']['train']['transform'][ 'value'] transforms_train, w, e = RetinaNet._verify_transforms( transforms_train, True) result['warnings'].extend(w) result['errors'].extend(e) if transforms_train is None: result['valid'] = False else: updatedOptions['options']['train']['transform'][ 'value'] = transforms_train transforms_inf = updatedOptions['options']['inference']['transform'][ 'value'] transforms_inf, w, e = RetinaNet._verify_transforms( transforms_inf, False) result['warnings'].extend(w) result['errors'].extend(e) if transforms_inf is None: result['valid'] = False else: updatedOptions['options']['inference']['transform'][ 'value'] = transforms_inf if result['valid']: result['options'] = updatedOptions return result