def __init__(self, project, config, dbConnector, fileServer, options): super(GenericDetectron2Model, self).__init__(project, config, dbConnector, fileServer, options) if isinstance(options, str): try: options = json.loads(options) except: # something went wrong; discard options #TODO options = None # try to fill and substitute global definitions in JSON-enhanced options if isinstance(options, dict): try: updatedOptions = optionsHelper.merge_options( self.getDefaultOptions(), options.copy()) updatedOptions = optionsHelper.substitute_definitions( updatedOptions) self.options = updatedOptions except: # something went wrong; ignore pass # verify options result = self.verifyOptions(self.options) if not result['valid']: print( f'[{self.project}] WARNING: provided options are invalid; replacing with defaults...' ) self.options = self.getDefaultOptions() # prepare Detectron2 configuration self.detectron2cfg = self._get_config()
def _load_default_options(cls, jsonFilePath, defaultOptions): try: # try to load defaults from JSON file first options = json.load(open(jsonFilePath, 'r')) except Exception as e: # error; fall back to built-in defaults print( f'Error reading default options file "{jsonFilePath}" (message: "{str(e)}"), falling back to built-in options.' ) options = defaultOptions # expand options options = optionsHelper.substitute_definitions(options) return options
def getDefaultOptions(): jsonFile = 'config/ai/model/pytorch/boundingBoxes/retinanet.json' try: # try to load defaults from JSON file first options = json.load(open(jsonFile, 'r')) except Exception as e: # error; fall back to built-in defaults print( f'Error reading default RetinaNet options file "{jsonFile}" (message: "{str(e)}"), falling back to built-in options.' ) options = DEFAULT_OPTIONS # expand options options = optionsHelper.substitute_definitions(options) return options
def verifyOptions(cls, options): defaultOptions = cls.getDefaultOptions() if options is None: return {'valid': True, 'options': defaultOptions} try: if isinstance(options, str): options = json.loads(options) options = optionsHelper.substitute_definitions(options) except Exception as e: return { 'valid': False, 'errors': [f'Options are not in a proper format (message: {str(e)}).'] } try: # mandatory field: model config modelConfig = optionsHelper.get_hierarchical_value( options, ['options', 'model', 'config', 'value']) if modelConfig is None: raise Exception('missing model type field in options.') opts, warnings, errors = optionsHelper.verify_options( options['options'], autoCorrect=True) options['options'] = opts return { 'valid': not len(errors), 'warnings': warnings, 'errors': errors, 'options': options } except Exception as e: return { 'valid': False, 'errors': [ f'An error occurred trying to verify options (message: {str(e)}).' ] }
def __init__(self, project, config, dbConnector, fileServer, options): super(GenericPyTorchModel, self).__init__(project, config, dbConnector, fileServer, options) # try to fill and substitute global definitions in JSON-enhanced options if isinstance(options, dict) and 'defs' in options: try: updatedOptions = optionsHelper.substitute_definitions( options.copy()) self.options = updatedOptions except: # something went wrong; ignore pass # retrieve executables try: self.model_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'model', 'class'])) except: self.model_class = None try: self.criterion_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'criterion', 'class'])) except: self.criterion_class = None try: self.optim_class = get_class_executable( optionsHelper.get_hierarchical_value( self.options, ['options', 'train', 'optim'])) except: self.optim_class = SGD try: self.dataset_class = get_class_executable( optionsHelper.get_hierarchical_value(self.options, ['options', 'dataset'])) except: self.dataset_class = None
def initializeTransforms(self, mode='train', options=None): ''' AIDE's Detectron2-compliant models all support the same transforms and thus can be initialized the same way. "mode" determines whether the transforms specified for 'train' or for 'inference' are to be initialized. If "options" contains a dict of AIDE model options, the transforms are to be initialized from there; otherwise the current class-specific ones are used. ''' assert mode in ('train', 'inference'), 'Invalid transform mode specified' if isinstance(options, dict): opt = copy.deepcopy(options) else: opt = copy.deepcopy(self.options) opt = optionsHelper.substitute_definitions(opt) # parse transforms transforms = [] #TODO: add a resize transform in any case transformOpts = optionsHelper.get_hierarchical_value( opt, ['options', mode, 'transform', 'value']) for tr in transformOpts: trClass = tr['id'] args = optionsHelper.filter_reserved_children(tr, True) for a in args.keys(): args[a] = optionsHelper.get_hierarchical_value( args[a], ['value']) if a == 'interp': #TODO: ugly solution to convert interpolation methods args[a] = int(args[a]) # initialize transform = getattr(T, trClass)(**args) transforms.append(transform) return transforms
def verifyOptions(options): # get default options to compare to defaultOptions = RetinaNet.getDefaultOptions() # updated options with modifications made if options is None: updatedOptions = defaultOptions.copy() else: if not isinstance(options, dict): try: options = json.loads(options) except Exception as e: return { 'valid': False, 'warnings': [], 'errors': [ f'Options are not in valid JSON format (message: "{str(e)}").' ] } updatedOptions = options.copy() result = {'valid': True, 'warnings': [], 'errors': []} if not 'defs' in updatedOptions: # old version (without GUI formatting): convert first updatedOptions, warnings = RetinaNet._convertOldOptions( updatedOptions, defaultOptions) result['warnings'].append( 'Options have been converted to new format.') result['warnings'].extend(warnings) # flatten and fill globals updatedOptions = optionsHelper.substitute_definitions(updatedOptions) # do the verification missingClassOptions = optionsHelper.get_hierarchical_value( updatedOptions, ['options', 'general', 'labelClasses']) if not isinstance(missingClassOptions, dict): updatedOptions['options']['general']['labelClasses'] = \ optionsHelper.get_hierarchical_value(defaultOptions, ['options', 'general', 'labelClasses']) #TODO: verify rest # verify transforms transforms_train = updatedOptions['options']['train']['transform'][ 'value'] transforms_train, w, e = RetinaNet._verify_transforms( transforms_train, True) result['warnings'].extend(w) result['errors'].extend(e) if transforms_train is None: result['valid'] = False else: updatedOptions['options']['train']['transform'][ 'value'] = transforms_train transforms_inf = updatedOptions['options']['inference']['transform'][ 'value'] transforms_inf, w, e = RetinaNet._verify_transforms( transforms_inf, False) result['warnings'].extend(w) result['errors'].extend(e) if transforms_inf is None: result['valid'] = False else: updatedOptions['options']['inference']['transform'][ 'value'] = transforms_inf if result['valid']: result['options'] = updatedOptions return result